Download
Link
collectorforkubernetes-opensearch.yaml
CURL
1curl -O https://www.outcoldsolutions.com/docs/elasticsearch-kubernetes/collectorforkubernetes-opensearch.yamlWGET
1wget https://www.outcoldsolutions.com/docs/elasticsearch-kubernetes/collectorforkubernetes-opensearch.yamlcollectorforkubernetes.yaml
1apiVersion: v1
2kind: Namespace
3metadata:
4 labels:
5 app: collectorforkubernetes
6 name: collectorforkubernetes
7---
8apiVersion: apiextensions.k8s.io/v1
9kind: CustomResourceDefinition
10metadata:
11 name: configurations.collectord.io
12spec:
13 group: collectord.io
14 versions:
15 - name: v1
16 served: true
17 storage: true
18 schema:
19 openAPIV3Schema:
20 type: object
21 properties:
22 spec:
23 type: object
24 additionalProperties: true
25 force:
26 type: boolean
27 scope: Cluster
28 names:
29 listKind: ConfigurationList
30 plural: configurations
31 singular: configuration
32 kind: Configuration
33---
34apiVersion: apiextensions.k8s.io/v1
35kind: CustomResourceDefinition
36metadata:
37 name: splunkoutputs.collectord.io
38spec:
39 group: collectord.io
40 versions:
41 - name: v1
42 served: true
43 storage: true
44 schema:
45 openAPIV3Schema:
46 type: object
47 properties:
48 spec:
49 type: object
50 properties:
51 url:
52 type: string
53 format: uri
54 insecure:
55 type: boolean
56 token:
57 type: string
58 description: "Plain token"
59 tokenFromSecret:
60 type: object
61 description: "Reference to a Kubernetes Secret"
62 properties:
63 secret:
64 type: string
65 key:
66 type: string
67 defaultMetricsIndex:
68 type: string
69 description: "Index of the metrics-index records sent through this output; empty for the token's default"
70 incorrectIndexBehavior:
71 type: string
72 description: "What happens to a record the token refuses"
73 enum: ["RedirectToDefault", "Drop", "Retry"]
74 requireExplicitIndex:
75 type: boolean
76 description: "Drop a record that carries no index instead of sending it to the token's default"
77 oneOf:
78 - required: ["token"]
79 - required: ["tokenFromSecret"]
80 scope: Namespaced
81 names:
82 listKind: SplunkOutputList
83 plural: splunkoutputs
84 singular: splunkoutput
85 kind: SplunkOutput
86---
87apiVersion: v1
88kind: ServiceAccount
89metadata:
90 labels:
91 app: collectorforkubernetes
92 name: collectorforkubernetes
93 namespace: collectorforkubernetes
94---
95apiVersion: rbac.authorization.k8s.io/v1
96kind: ClusterRole
97metadata:
98 name: collectorforkubernetes
99 labels:
100 app: collectorforkubernetes
101rules:
102 - apiGroups:
103 - ""
104 resources:
105 - events
106 - namespaces
107 - nodes
108 - nodes/metrics
109 - persistentvolumeclaims
110 - persistentvolumes
111 - pods
112 - replicationcontrollers
113 - resourcequotas
114 - services
115 verbs:
116 - get
117 - list
118 - watch
119 - apiGroups:
120 - apps
121 resources:
122 - daemonsets
123 - deployments
124 - replicasets
125 - statefulsets
126 verbs:
127 - get
128 - list
129 - watch
130 - apiGroups:
131 - batch
132 resources:
133 - cronjobs
134 - jobs
135 verbs:
136 - get
137 - list
138 - watch
139 - apiGroups:
140 - discovery.k8s.io
141 resources:
142 - endpointslices
143 verbs:
144 - get
145 - list
146 - watch
147 - apiGroups:
148 - rbac.authorization.k8s.io
149 resources:
150 - clusterroles
151 verbs:
152 - get
153 - list
154 - watch
155 - apiGroups:
156 - collectord.io
157 resources:
158 - configurations
159 - splunkoutputs
160 verbs:
161 - get
162 - list
163 - watch
164 - apiGroups: []
165 nonResourceURLs:
166 - /metrics
167 resources: []
168 verbs:
169 - get
170 - apiGroups: []
171 nonResourceURLs:
172 - /api
173 - /api/*
174 - /apis
175 - /apis/*
176 resources: []
177 verbs:
178 - get
179---
180apiVersion: rbac.authorization.k8s.io/v1
181kind: ClusterRoleBinding
182metadata:
183 name: collectorforkubernetes
184 labels:
185 app: collectorforkubernetes
186roleRef:
187 apiGroup: rbac.authorization.k8s.io
188 kind: ClusterRole
189 name: collectorforkubernetes
190subjects:
191 - kind: ServiceAccount
192 name: collectorforkubernetes
193 namespace: collectorforkubernetes
194---
195apiVersion: v1
196kind: ResourceQuota
197metadata:
198 name: collectorforkubernetes-critical-pods
199 namespace: collectorforkubernetes
200 labels:
201 app: collectorforkubernetes
202spec:
203 hard:
204 pods: 1G
205 scopeSelector:
206 matchExpressions:
207 - scopeName: PriorityClass
208 operator: In
209 values:
210 - system-cluster-critical
211 - system-node-critical
212---
213apiVersion: v1
214kind: ConfigMap
215metadata:
216 name: collectorforkubernetes-elasticsearch
217 namespace: collectorforkubernetes
218 labels:
219 app: collectorforkubernetes-elasticsearch
220data:
221 001-general.conf: |222 # The general configuration is used for all deployments
223 #
224 # Run collectord with the flag -conf and specify location of the configuration files.
225 #
226 # You can override all the values using environment variables with the format like
227 # COLLECTOR__<ANYNAME>=<section>__<key>=<value>
228 # As an example you can set dataPath in [general] section as
229 # COLLECTOR__DATAPATH=general__dataPath=C:\\some\\path\\data.db
230 # This parameter can be configured using -env-override, set it to empty string to disable this feature
231
232 [general]
233
234 # Please review license https://www.outcoldsolutions.com/legal/license-agreement/
235 # and accept license by changing the value to *true*
236 acceptLicense = false
237
238 # Location for the database
239 # Collectord stores positions of the files and internal state
240 dataPath = ./data/
241
242 # log level (accepted values are trace, debug, info, warn, error, fatal)
243 logLevel = info
244
245 # Internal http server - serves the liveness (/healthz) and readiness (/readyz)
246 # probes. DaemonSets run on the host network, so bind to loopback (127.0.0.1:11888)
247 # to keep the endpoints off the node IP; the kubelet still reaches them via
248 # httpGet.host 127.0.0.1. The addon runs off the host network and overrides the
249 # bind to 0.0.0.0 in 004-addon.conf so the kubelet can reach it on the pod IP.
250 # httpServerBinding = 0.0.0.0:11888
251 httpServerBinding = 127.0.0.1:11892
252 httpServerEndpoints.health = true
253 httpServerEndpoints.ready = true
254 httpServerEndpoints.metrics = false
255 httpServerEndpoints.pprof = false
256
257 httpServerLog = false
258
259 # telemetry report endpoint, set it to empty string to disable telemetry
260 telemetryEndpoint = https://license.outcold.solutions/telemetry/
261
262 licenseEndpoint = https://license.outcold.solutions/license/
263
264 # Only for the Outcold Solutions License Server; one on-premises uses [license.client]
265 licenseServerProxyUrl =
266
267 # user:password
268 licenseServerProxyBasicAuth =
269
270 license =
271
272 # Environment variable $KUBERNETES_NODENAME is used by default to setup hostname
273 # Use value below to override specific name
274 hostname =
275
276 # Default output for events, logs and metrics
277 # valid values: splunk and devnull
278 # Use devnull by default if you don't want to redirect data
279 defaultOutput = elasticsearch
280
281 fileInputBufferSize = 8kb
282
283 fileInputLineMaxSize = 1mb
284
285 # Include custom fields to attach to every event, in example below every event sent to Splunk will hav
286 # indexed field my_environment=dev. Fields names should match to ^[a-z][_a-z0-9]*$
287 # Better way to configure that is to specify labels for Kubernetes Nodes.
288 # ; fields.my_environment = dev
289 # Identify the cluster if you are planning to monitor multiple clusters
290 fields.agent.ephemeral_id = ${COLLECTORD_INSTANCE_RUNTIME_ID}
291 fields.agent.id = ${COLLECTORD_INSTANCE_ID}
292 fields.agent.name = ${KUBERNETES_NODENAME}
293 fields.agent.type = collectord
294 fields.agent.version = ${COLLECTORD_VERSION}
295 fields.ecs.version = 8.0.0
296 fields.host.architecture = ${COLLECTORD_ARCH}
297 fields.host.hostname = ${KUBERNETES_NODENAME}
298 fields.host.name = ${KUBERNETES_NODENAME}
299 fields.orchestrator.cluster.name = -
300 fields.orchestrator.type = kubernetes
301
302 # Include EC2 Metadata (see list of possible fields https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
303 # Should be in format ec2Metadata.{desired_field_name} = {url path to read the value}
304 # ec2Metadata.ec2_instance_id = /latest/meta-data/instance-id
305 # ec2Metadata.ec2_instance_type = /latest/meta-data/instance-type
306
307 # subdomain for the annotations added to the pods, workloads, namespaces or containers, like splunk.collectord.io/..
308 annotationsSubdomain = opensearch
309
310 # configure global thruput per second for forwarded logs (metrics are not included)
311 # for example if you set `thruputPerSecond = 512Kb`, that will limit amount of logs forwarded
312 # from the single Collectord instance to 512Kb per second.
313 # You can configure thruput individually for the logs (including specific for container logs) below
314 thruputPerSecond =
315
316 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
317 # older than 7 days
318 tooOldEvents =
319
320 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
321 tooNewEvents =
322
323 # How Collectord spells the fields it composes: "_" (kubernetes_pod_name, the default) or "."
324 fieldsSeparator =
325
326 # For input.files::X and application logs, when glob or match are configured, Collectord can automatically
327 # detect gzipped files and skip them (based on the extensions or magic numbers)
328 autoSkipGzipFiles = true
329
330 # Publish to non-default outputs asynchronously (default), so a slow or down output does not
331 # block the others; false holds the pipeline back instead of dropping events.
332 multioutput.async = true
333
334 # Buffer size for the async proxy (default 100), per output of each pipeline. When it and
335 # the output's queue are full, events are dropped (multioutput_dropped_events).
336 multioutput.asyncBufferSize = 100
337
338 [license.client]
339 # point to the license located on the HTTP web server, or a hosted by the Collectord running as license server
340 url =
341 basicAuth =
342 insecure = false
343 capath =
344 caname =
345 proxyUrl =
346
347 # user:password
348 proxyBasicAuth =
349
350 [general.kubernetes]
351
352 # Override service URL for Kubernetes (default is ${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT})
353 serviceURL =
354
355 # Environment variable $KUBERNETES_NODENAME is used by default to setup nodeName
356 # Use it only when you need to override it
357 nodeName =
358
359 tokenPath = /var/run/secrets/kubernetes.io/serviceaccount/token
360 certPath = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
361
362 # Default timeout for http responses. The streaming/watch requests depend on this timeout.
363 timeout = 30m
364
365 # How long to keep the cache for the recent calls to API server (to limit number of calls when collectord discovers new pods)
366 metadataTTL = 30s
367
368 podsCgroupFilter = ^/([^/\s]+/)*kubepods(\.slice)?/((kubepods-)?(burstable|besteffort)(\.slice)?/)?([^/]*)pod([0-9a-f]{32}|[0-9a-f\-_]{36})(\.slice)?$
369
370 containersCgroupFilter = ^/([^/\s]+/)*kubepods(\.slice)?/((kubepods-)?(burstable|besteffort)(\.slice)?/)?([^/]*)pod([0-9a-f]{32}|[0-9a-f\-_]{36})(\.slice)?/(docker-|crio-|cri-\w+-)?[0-9a-f]{64}(\.scope)?(\/.+)?$
371
372 # path to the kubelet root location (use it to discover application logs for emptyDir)
373 # the expected format is `pods/{pod-id}/volumes/kubernetes.io~empty-dir/{volume-name}/_data/`
374 volumesRootDir = /rootfs/var/lib/kubelet/
375
376 # You can attach annotations as a metadata, using the format
377 # includeAnnotations.{key} = {regexp}
378 # For example if you want to include all annotations that starts with `prometheus.io` or `example.com` you can include
379 # the following format:
380 # includeAnnotations.1 = ^prometheus\.io.*
381 # includeAnnotations.2 = ^example\.com.*
382
383 # You can exclude labels from metadata, using the format
384 # excludeLabels.{key} = {regexp}
385 # For example if you want to exclude all labels that starts with `prometheus.io` or `example.com` you can include
386 # the following format:
387 # excludeLabels.1 = ^prometheus\.io.*
388 # excludeLabels.2 = ^example\.com.*
389
390 # watch for changes (annotations) in the objects
391 watch.namespace = v1/namespace
392 watch.deployment = apps/v1/deployment
393 watch.configuration = collectord.io/v1/configuration
394
395 # Collectord can review the assigned ClusterRole and traverse metadata for the Pods only for the Owner objects
396 # that are defined in the ClusterRole, ignoring anything else, it does not have access to.
397 # This way Collectord does not generate 403 requests on API Server
398 clusterRole = collectorforkubernetes
399
400 # Alternative of telling Collectord about the ClusterRole is to manually list the objects.
401 # You can define which objects Collectord should traverse when it sees Owners.
402 ; traverseOwnership.namespaces = v1/namespace
403
404 [output.elasticsearch]
405
406 dataStream = logs-collectord-{{agent.version}}
407 dataStreamFailedEvents = logs-collectord-failed-{{agent.version}}
408
409 # ElasticSearch Scheme Host and Port, with an optional path every request goes under
410 # (https://gateway/elasticsearch, behind a reverse proxy)
411 host =
412
413 # Specify how Hosts should be picked up (in case if multiple is used)
414 # * random - choose random url on first selection and a different random one after each failure (connection or HTTP status code >= 500)
415 # * round-robin - choose url starting from first one and bump on each failure (connection or HTTP status code >= 500)
416 # * random-with-round-robin - choose random url on first selection and after that in round-robin on each
417 # failure (connection or HTTP status code >= 500)
418 # With round-robin and random-with-round-robin, the instances on a failed url can all move to the same next one.
419 hostSelection = random
420 authorizationBasicUsername =
421 authorizationBasicPassword =
422 headers.Accept = application/json
423 headers.Content-Type = application/json
424 insecure = false
425 caPath =
426 caName =
427 clientCertPath =
428 clientKeyPath =
429
430 # Events are batched with the maximum size set by batchSize and staying in a pipeline for not longer
431 # than set by frequency
432 frequency = 5s
433 batchSize = 768K
434
435 # limit by the number of events (0 value has no limit on the number of events)
436 events = 50
437 proxyUrl =
438
439 # user:password
440 proxyBasicAuth =
441
442 # Timeout specifies a time limit for requests made by collectord.
443 # The timeout includes connection time, any
444 # redirects, and reading the response body.
445 timeout = 30s
446
447 # gzip compression level (nocompression, default, 1...9)
448 compressionLevel = default
449
450 # number of dedicated elasticsearch output threads (to increase throughput above 4k events per second)
451 threads = 2
452
453 # Default algorithm between threads is roundrobin, but you can change it to weighted
454 threadsAlgorithm = weighted
455
456 # Submit objects to elasticsearch
457 put._index_template/logs-collectord-${COLLECTORD_VERSION} = /config/es-default-index-template.json
458 put._index_template/logs-collectord-failed-${COLLECTORD_VERSION} = /config/es-failed-index-template.json
459
460 # Dedicated queue size for the output, default is 1024, larger queue sizes will require more memory,
461 # but will allow to handle more events in case of network issues
462 queueSize = 1024463 002-daemonset.conf: | 464 [general.cri-o]
465
466 # url for CRIO API, only unix socket is supported
467 url = unix:///rootfs/var/run/crio/crio.sock
468
469 # Timeout for http responses to docker client. The streaming requests depend on this timeout.
470 timeout = 1m
471
472 [general.containerd]
473 # Runtime can be on /rootfs/run/containerd (depends on the Linux distribution)
474 runtimePath = /rootfs/var/run/containerd
475 namespace = k8s.io
476
477 [input.files]
478
479 disabled = false
480
481 # root location of docker log files
482 # logs are expected in standard docker format like {containerID}/{containerID}-json.log
483 # rotated files
484 path = /rootfs/var/lib/docker/containers/
485
486 # root location of CRI-O files
487 # logs are expected in Kubernetes format, like {podID}/{containerName}/0.log
488 crioPath = /rootfs/var/log/pods/
489
490 # files are read using polling schema, when reach the EOF how often to check if files got updated
491 pollingInterval = 250ms
492
493 # how often to look for the new files under logs path
494 walkingInterval = 5s
495
496 # include verbose fields in events (file offset)
497 verboseFields = false
498
499 type = container
500
501 elasticsearch.datastream =
502
503 # docker splits events when they are larger than 10-100k (depends on the docker version)
504 # we join them together by default and forward to Splunk as one event
505 joinPartialEvents = true
506
507 # In case if your containers report messages with terminal colors or other escape sequences
508 # you can enable strip for all the containers in one place.
509 # Better is to enable it only for required container with the annotation opensearch.collectord.io/logs-escapeterminalsequences=true
510 stripTerminalEscapeSequences = false
511
512 # Regexp used for stripping terminal colors, it does not stip all the escape sequences
513 # Read http://man7.org/linux/man-pages/man4/console_codes.4.html for more information
514 stripTerminalEscapeSequencesRegex = (\x1b\[\d{1,3}(;\d{1,3})*m)|(\x07)|(\x1b]\d+(\s\d)?;[^\x07]+\x07)|(.*\x1b\[K)
515
516 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
517 samplingPercent = -1
518
519 # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
520 samplingKey =
521
522 output =
523
524 # configure default thruput per second for for each container log
525 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
526 # from the single container to 128Kb per second.
527 thruputPerSecond =
528
529 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
530 # older than 7 days
531 tooOldEvents =
532
533 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
534 tooNewEvents =
535
536 # Read rotated logs the kubelet compressed before Collectord read them; false counts them lost
537 readCompressed = true
538
539 # Bytes of those logs inflated at once, node-wide, under [general] dataPath
540 compressedInflateBudget = 256MiB
541
542 # Group each stream's repeated lines into one event per run that counts them.
543 # Off unless true; a workload overrides it with its
544 # opensearch.collectord.io/logs-repeat-enabled annotation.
545 repeatEnabled = false
546
547 # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
548 # repeatMaxSpan = 1m
549
550 # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
551 # repeatOffsets = false
552
553 # What each stream may remember: distinct lines, lines one event stands for, and bytes
554 # repeatMaxKeys = 256
555 # repeatMaxCount = 1000
556 # repeatMaxBytes = 256KB
557
558 # Fold each stream's lines of one pattern past foldMaxPerMinute into one summary a minute.
559 # Lossy: folded lines are not sent. Off unless true; a workload overrides it with its
560 # opensearch.collectord.io/logs-fold-enabled annotation.
561 foldEnabled = false
562
563 # Lines of one pattern a minute sent before the rest fold
564 # foldMaxPerMinute = 60
565
566 # Lines never folded (a regular expression; empty: the default)
567 # foldKeep = (?i)error|exception|fatal|panic
568
569 # What each stream may keep a minute: its patterns and summaries
570 # foldMaxBytes = 256KB
571
572 # Report each stream's top patterns every interval, as events of their own.
573 # Off unless true; a workload overrides it with its
574 # opensearch.collectord.io/logs-patterns-enabled annotation.
575 patternsEnabled = false
576
577 # How often a report goes out, from 1m to 24h
578 # patternsInterval = 15m
579
580 # How many patterns a report carries, up to 100
581 # patternsTop = 20
582
583 # What each stream's patterns may keep; past it a new one takes the place of the lightest
584 # patternsMaxBytes = 256KB
585
586 # The report events' type (empty: the input's type with _patterns after it)
587 # patternsType =
588
589 [input.app_logs]
590
591 disabled = false
592
593 # root location of mounts (applies to hostPath mounts only), if the hostPath differs inside container from the path on host
594 root = /rootfs/
595
596 # how often to review list of available volumes
597 syncInterval = 5s
598
599 glob = *.log*
600
601 # files are read using polling schema, when reach the EOF how often to check if files got updated
602 pollingInterval = 250ms
603
604 # how often to look for the new files under logs path
605 walkingInterval = 5s
606
607 # include verbose fields in events (file offset)
608 verboseFields = false
609
610 type = container.file
611
612 elasticsearch.datastream =
613
614 # we split files using new line character, with this configuration you can specify what defines the new event
615 # after new line
616 eventPatternRegex = ^[^\s]
617 eventPatternMaxInterval = 100ms
618 eventPatternMaxWait = 1s
619 eventPatternMaxSize = 1MB
620
621 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
622 samplingPercent = -1
623
624 # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
625 samplingKey =
626
627 output =
628
629 # configure default thruput per second for for each container log
630 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
631 # from the single container to 128Kb per second.
632 thruputPerSecond =
633
634 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
635 # older than 7 days
636 tooOldEvents =
637
638 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
639 tooNewEvents =
640
641 # Configure how long Collectord should keep the file descriptors open for files, that has not been forwarded yet
642 # When using PVC, and if pipeline is lagging behind, Collectord holding open fd for files, can cause long termination
643 # of pods, as kubelet cannot unmount the PVC volume from the system
644 maxHoldAfterClose = 1800s
645
646 # Group each file's repeated lines into one event per run that counts them.
647 # Off unless true; a workload overrides it with its
648 # opensearch.collectord.io/volume.<n>-logs-repeat-enabled annotation.
649 repeatEnabled = false
650
651 # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
652 # repeatMaxSpan = 1m
653
654 # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
655 # repeatOffsets = false
656
657 # What each file may remember: distinct lines, lines one event stands for, and bytes
658 # repeatMaxKeys = 256
659 # repeatMaxCount = 1000
660 # repeatMaxBytes = 256KB
661
662 # Fold each file's lines of one pattern past foldMaxPerMinute into one summary a minute.
663 # Lossy: folded lines are not sent. Off unless true; a workload overrides it with its
664 # opensearch.collectord.io/volume.<n>-logs-fold-enabled annotation.
665 foldEnabled = false
666
667 # Lines of one pattern a minute sent before the rest fold
668 # foldMaxPerMinute = 60
669
670 # Lines never folded (a regular expression; empty: the default)
671 # foldKeep = (?i)error|exception|fatal|panic
672
673 # What each file may keep a minute: its patterns and summaries
674 # foldMaxBytes = 256KB
675
676 # Report each file's top patterns every interval, as events of their own.
677 # Off unless true; a workload overrides it with its
678 # opensearch.collectord.io/volume.<n>-logs-patterns-enabled annotation.
679 patternsEnabled = false
680
681 # How often a report goes out, from 1m to 24h
682 # patternsInterval = 15m
683
684 # How many patterns a report carries, up to 100
685 # patternsTop = 20
686
687 # What each file's patterns may keep; past it a new one takes the place of the lightest
688 # patternsMaxBytes = 256KB
689
690 # The report events' type (empty: the input's type with _patterns after it)
691 # patternsType =
692
693 [input.files::audit-logs]
694
695 disabled = false
696
697 path = /rootfs/var/log/kube-apiserver/
698
699 glob = audit*.log
700
701 match =
702
703 # limit search only on one level
704 recursive = false
705 nodeSelector = node-role.kubernetes.io/control-plane
706
707 # files are read using polling schema, when reach the EOF how often to check if files got updated
708 pollingInterval = 250ms
709
710 # how often o look for the new files under logs path
711 walkingInterval = 5s
712
713 # include verbose fields in events (file offset)
714 verboseFields = false
715
716 type = file
717
718 elasticsearch.datastream =
719
720 extraction = (?P<message>.*"stageTimestamp":"(?P<timestamp>[^"]+)".*)
721 extractionMessageField = message
722
723 timestampField = timestamp
724
725 # format for timestamp
726 # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006`
727 timestampFormat = 2006-01-02T15:04:05.999999999Z07:00
728
729 # timestamp location (if not defined by format)
730 timestampLocation =
731
732 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
733 samplingPercent = -1
734
735 # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
736 samplingKey =
737
738 output =
739
740 # configure default thruput per second for this files group
741 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
742 # from the files in this group to 128Kb per second.
743 thruputPerSecond =
744
745 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
746 # older than 7 days
747 tooOldEvents =
748
749 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
750 tooNewEvents =
751
752 # by default every new event should start from not space symbol
753 eventPattern =
754
755 # Blacklisting and whitelisting the logs
756 # whitelist.0 = ^regexp$
757 # blacklist.0 = ^regexp$
758
759 # Lines matching a thruputExempt pattern pass every thruput cap and sampling; with one set,
760 # a spent cap drops the other lines instead of pausing this input
761 # thruputExempt.0 = ^regexp$
762
763 # Group each file's repeated lines into one event per run that counts them.
764 # Off unless true.
765 repeatEnabled = false
766
767 # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
768 # repeatMaxSpan = 1m
769
770 # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
771 # repeatOffsets = false
772
773 # What each file may remember: distinct lines, lines one event stands for, and bytes
774 # repeatMaxKeys = 256
775 # repeatMaxCount = 1000
776 # repeatMaxBytes = 256KB
777
778 # Fold each file's lines of one pattern past foldMaxPerMinute into one summary a minute.
779 # Lossy: folded lines are not sent. Off unless true.
780 foldEnabled = false
781
782 # Lines of one pattern a minute sent before the rest fold
783 # foldMaxPerMinute = 60
784
785 # Lines never folded (a regular expression; empty: the default)
786 # foldKeep = (?i)error|exception|fatal|panic
787
788 # What each file may keep a minute: its patterns and summaries
789 # foldMaxBytes = 256KB
790
791 # Report each file's top patterns every interval, as events of their own.
792 # Off unless true.
793 patternsEnabled = false
794
795 # How often a report goes out, from 1m to 24h
796 # patternsInterval = 15m
797
798 # How many patterns a report carries, up to 100
799 # patternsTop = 20
800
801 # What each file's patterns may keep; past it a new one takes the place of the lightest
802 # patternsMaxBytes = 256KB
803
804 # The report events' type (empty: the input's type with _patterns after it)
805 # patternsType =
806
807 [input.files::logs]
808
809 disabled = false
810
811 path = /rootfs/var/log/
812
813 glob =
814
815 match = ^(([\w\-.]+\.log(.[\d\-]+)?)|(docker))$
816 exclude = ^(auth|cron|daemon|kern|lpr|mail|user|ufw)\.log(.[\d\-]+)?$
817
818 # limit search only on one level
819 recursive = false
820
821 # files are read using polling schema, when reach the EOF how often to check if files got updated
822 pollingInterval = 250ms
823
824 # how often o look for the new files under logs path
825 walkingInterval = 5s
826
827 # include verbose fields in events (file offset)
828 verboseFields = false
829
830 type = file
831
832 elasticsearch.datastream =
833
834 extraction =
835 extractionMessageField =
836
837 timestampField =
838
839 # format for timestamp
840 # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006`
841 timestampFormat =
842
843 # Adjust date, if month/day aren't set in format
844 timestampSetMonth = false
845 timestampSetDay = false
846
847 # timestamp location (if not defined by format)
848 timestampLocation =
849
850 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
851 samplingPercent = -1
852
853 # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
854 samplingKey =
855
856 output =
857
858 # configure default thruput per second for this files group
859 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
860 # from the files in this group to 128Kb per second.
861 thruputPerSecond =
862
863 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
864 # older than 7 days
865 tooOldEvents =
866
867 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
868 tooNewEvents =
869
870 # by default every new event should start from not space symbol
871 eventPattern = ^[^\s]
872
873 # Blacklisting and whitelisting the logs
874 # whitelist.0 = ^regexp$
875 # blacklist.0 = ^regexp$
876
877 # Lines matching a thruputExempt pattern pass every thruput cap and sampling; with one set,
878 # a spent cap drops the other lines instead of pausing this input
879 # thruputExempt.0 = ^regexp$
880
881 # Group each file's repeated lines into one event per run that counts them.
882 # Off unless true.
883 repeatEnabled = false
884
885 # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
886 # repeatMaxSpan = 1m
887
888 # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
889 # repeatOffsets = false
890
891 # What each file may remember: distinct lines, lines one event stands for, and bytes
892 # repeatMaxKeys = 256
893 # repeatMaxCount = 1000
894 # repeatMaxBytes = 256KB
895
896 # Fold each file's lines of one pattern past foldMaxPerMinute into one summary a minute.
897 # Lossy: folded lines are not sent. Off unless true.
898 foldEnabled = false
899
900 # Lines of one pattern a minute sent before the rest fold
901 # foldMaxPerMinute = 60
902
903 # Lines never folded (a regular expression; empty: the default)
904 # foldKeep = (?i)error|exception|fatal|panic
905
906 # What each file may keep a minute: its patterns and summaries
907 # foldMaxBytes = 256KB
908
909 # Report each file's top patterns every interval, as events of their own.
910 # Off unless true.
911 patternsEnabled = false
912
913 # How often a report goes out, from 1m to 24h
914 # patternsInterval = 15m
915
916 # How many patterns a report carries, up to 100
917 # patternsTop = 20
918
919 # What each file's patterns may keep; past it a new one takes the place of the lightest
920 # patternsMaxBytes = 256KB
921
922 # The report events' type (empty: the input's type with _patterns after it)
923 # patternsType =
924
925 [input.files::syslog]
926
927 disabled = true
928
929 path = /rootfs/var/log/
930
931 glob =
932
933 match = ^(syslog|messages|auth\.log)(.[\d\-]+)?$
934
935 # limit search only on one level
936 recursive = false
937
938 # files are read using polling schema, when reach the EOF how often to check if files got updated
939 pollingInterval = 250ms
940
941 # how often o look for the new files under logs path
942 walkingInterval = 5s
943
944 # include verbose fields in events (file offset)
945 verboseFields = false
946
947 type = file
948
949 elasticsearch.datastream =
950
951 extraction = ^(?P<timestamp>[A-Za-z]+\s+\d+\s\d+:\d+:\d+|\d{4}-\d{2}-\d{2}T\S+)\s(?P<log__syslog__hostname>[^\s]+)\s(?P<log__syslog__appname>[^:\[]+)(\[(?P<log__syslog__procid>\d+)\])?: (.+)$
952 extractionMessageField =
953
954 # the extracted fields naming the process that wrote a line, which groups its lines apart
955 extractionWriterFields = log__syslog__hostname, log__syslog__appname, log__syslog__procid
956
957 timestampField = timestamp
958
959 # format for timestamp
960 # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006`
961 timestampFormat = @syslog
962
963 # Adjust date, if month/day aren't set in format
964 timestampSetMonth = false
965 timestampSetDay = false
966
967 # timestamp location (if not defined by format)
968 timestampLocation = Local
969
970 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
971 samplingPercent = -1
972
973 # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
974 samplingKey =
975
976 output =
977
978 # configure default thruput per second for this files group
979 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
980 # from the files in this group to 128Kb per second.
981 thruputPerSecond =
982
983 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
984 # older than 7 days
985 tooOldEvents =
986
987 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
988 tooNewEvents =
989
990 # by default every new event should start from not space symbol
991 eventPattern = ^[^\s]
992
993 # Blacklisting and whitelisting the logs
994 # whitelist.0 = ^regexp$
995 # blacklist.0 = ^regexp$
996
997 # Lines matching a thruputExempt pattern pass every thruput cap and sampling; with one set,
998 # a spent cap drops the other lines instead of pausing this input
999 # thruputExempt.0 = ^regexp$
1000
1001 # Group each file's repeated lines into one event per run that counts them.
1002 # Off unless true.
1003 repeatEnabled = false
1004
1005 # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
1006 # repeatMaxSpan = 1m
1007
1008 # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
1009 # repeatOffsets = false
1010
1011 # What each file may remember: distinct lines, lines one event stands for, and bytes
1012 # repeatMaxKeys = 256
1013 # repeatMaxCount = 1000
1014 # repeatMaxBytes = 256KB
1015
1016 # Fold each file's lines of one pattern past foldMaxPerMinute into one summary a minute.
1017 # Lossy: folded lines are not sent. Off unless true.
1018 foldEnabled = false
1019
1020 # Lines of one pattern a minute sent before the rest fold
1021 # foldMaxPerMinute = 60
1022
1023 # Lines never folded (a regular expression; empty: the default)
1024 # foldKeep = (?i)error|exception|fatal|panic
1025
1026 # What each file may keep a minute: its patterns and summaries
1027 # foldMaxBytes = 256KB
1028
1029 # Report each file's top patterns every interval, as events of their own.
1030 # Off unless true.
1031 patternsEnabled = false
1032
1033 # How often a report goes out, from 1m to 24h
1034 # patternsInterval = 15m
1035
1036 # How many patterns a report carries, up to 100
1037 # patternsTop = 20
1038
1039 # What each file's patterns may keep; past it a new one takes the place of the lightest
1040 # patternsMaxBytes = 256KB
1041
1042 # The report events' type (empty: the input's type with _patterns after it)
1043 # patternsType =
1044
1045 [input.journald]
1046
1047 disabled = false
1048
1049 path.persistent = /rootfs/var/log/journal/
1050 path.volatile = /rootfs/run/log/journal/
1051
1052 # when reach end of journald, how often to pull
1053 pollingInterval = 250ms
1054
1055 # if you don't want to forward journald from the beginning,
1056 # set the oldest event in relative value, like -14h or -30m or -30s (h/m/s supported)
1057 startFromRel =
1058
1059 type = journald
1060
1061 elasticsearch.datastream =
1062
1063 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
1064 samplingPercent = -1
1065
1066 # sampling key (should be regexp with the named match pattern `key`)
1067 samplingKey =
1068
1069 # how often to reopen the journald to free old files
1070 reopenInterval = 1h
1071
1072 output =
1073
1074 # configure default thruput per second for journald
1075 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
1076 # from the journald to 128Kb per second.
1077 thruputPerSecond =
1078
1079 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
1080 # older than 7 days
1081 tooOldEvents =
1082
1083 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
1084 tooNewEvents =
1085
1086 # by default every new event should start from not space symbol
1087 eventPattern = ^[^\s]
1088
1089 # By default ignoring verbose hyperkube logs (all INFO messages)
1090
1091 # Lines matching a thruputExempt pattern pass every thruput cap and sampling; with one set,
1092 # a spent cap drops the other lines instead of pausing journald
1093
1094 # Move Journald logs reader to a separate process, to prevent process from crashing in case of corrupted log files
1095 spawnExternalProcess = false
1096
1097 # Group each writer's repeated lines into one event per run that counts them.
1098 # Off unless true.
1099 # A writer is a unit, identifier, command, process, priority and facility: never grouped with another.
1100 repeatEnabled = false
1101
1102 # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
1103 # repeatMaxSpan = 1m
1104
1105 # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
1106 # repeatOffsets = false
1107
1108 # What the journal, all its writers together, may remember: distinct lines, lines one event stands for, and bytes
1109 # repeatMaxKeys = 256
1110 # repeatMaxCount = 1000
1111 # repeatMaxBytes = 256KB
1112
1113 # Fold each writer's lines of one pattern past foldMaxPerMinute into one summary a minute.
1114 # Lossy: folded lines are not sent. Off unless true.
1115 # A writer is a unit, identifier, command, process, priority and facility: never folded with another.
1116 foldEnabled = false
1117
1118 # Lines of one pattern a minute sent before the rest fold
1119 # foldMaxPerMinute = 60
1120
1121 # Lines never folded (a regular expression; empty: the default)
1122 # foldKeep = (?i)error|exception|fatal|panic
1123
1124 # What the journal, all its writers together, may keep a minute: its patterns and summaries
1125 # foldMaxBytes = 256KB
1126
1127 # Report the journal's top patterns every interval, as events of their own.
1128 # Off unless true.
1129 # A writer is a unit, identifier, command, process, priority and facility: its lines never share a pattern with another's, and the top patterns are of all its writers together.
1130 patternsEnabled = false
1131
1132 # How often a report goes out, from 1m to 24h
1133 # patternsInterval = 15m
1134
1135 # How many patterns a report carries, up to 100
1136 # patternsTop = 20
1137
1138 # What the journal's patterns may keep; past it a new one takes the place of the lightest
1139 # patternsMaxBytes = 256KB
1140
1141 # The report events' type (empty: the input's type with _patterns after it)
1142 # patternsType =
1143
1144 # Pipe to join events (container logs only)
1145 [pipe.join]
1146
1147 disabled = false
1148
1149 maxInterval = 100ms
1150
1151 maxWait = 1s
1152
1153 maxSize = 1MB
1154
1155 # Default pattern to indicate new message (should start not from space)
1156 patternRegex = ^[^\s]
1157
1158 # You can configure global replace rules for the events, which can help to remove sensitive data
1159 # from logs before they are sent to outputs. Those rules will be applied to all pipelines for container logs, host logs,
1160 # application logs and events.
1161 # In the following example we replace password=TEST with password=********
1162 ; [pipe.replace::name]
1163 ; patternRegex = (password=)([^\s]+)
1164 ; replace = $1********
1165
1166 # You can configure global hash rules for the events, which can help to hide sensitive data
1167 # from logs before they are sent to outputs. Those rules will be applied to all pipelines for container logs, host logs,
1168 # application logs and events.
1169 # In the following example we hash IP addresses with fnv-1a-64
1170 ; [pipe.hash::name]
1171 ; match = (\d{1,3}\.){3}\d{1,3}'
1172 ; function = fnv-1a-64
1173
1174 # Log lines matching patternRegex pass every thruput cap and sampling, in the workloads whose
1175 # fields match every matchRegex.<field>; over its cap such a workload drops its other lines
1176 # instead of pausing. Select by namespace: pod, container and image names are the workload
1177 # owner's to choose, and label fields never match
1178 ; [pipe.thruput_exempt::payments-audit]
1179 ; matchRegex.kubernetes_namespace = ^payments$
1180 ; patternRegex = ^AUDIT
1181
1182 # Count the events and bytes leaving every pipeline (pipeline_events, pipeline_bytes);
1183 # maxSeries bounds the label combinations. Fields are written with underscores.
1184 ; [pipe.eventcount]
1185 ; labels = index, sourcetype, output
1186 ; fields = kubernetes_namespace_name
1187 ; maxSeries = 10001188 004-addon.conf: |1189 [general]
1190 # addons can be run in parallel with agents
1191 addon = true
1192
1193 # The addon runs off the host network, so the kubelet probes /healthz and /readyz
1194 # on the pod IP - bind 0.0.0.0 here (overrides the loopback default in
1195 # 001-general.conf), on the port that binding names.
1196 httpServerBinding = 0.0.0.0:11892
1197
1198 [input.kubernetes_events]
1199
1200 disabled = false
1201
1202 type = events
1203
1204 elasticsearch.datastream =
1205
1206 output =
1207
1208 excludeManagedFields = true
1209
1210 [input.kubernetes_watch::deployments]
1211 disabled = false
1212
1213 # How often every object is forwarded again; one no longer listed is forwarded as DELETED
1214 # with "reason":"absent_from_relist"
1215 refresh = 45m
1216
1217 apiVersion = apps/v1
1218 kind = deployment
1219 namespace =
1220
1221 type = objects
1222
1223 elasticsearch.datastream =
1224
1225 output =
1226
1227 excludeManagedFields = true
1228
1229 # you can remove or hash some values in the events (after modifyValues you can define path in the JSON object,
1230 # and the value can be hash:{hashFunction}, or remove to remove the object )
1231 ; modifyValues.object.data.* = hash:sha256
1232 ; modifyValues.object.metadata.annotations.* = remove
1233
1234 # You can exclude events by namespace with blacklist or whitelist only required namespaces
1235 # blacklist.kubernetes.namespace.name = ^namespace0$
1236 # whitelist.kubernetes.namespace.name = ^((namespace1)|(namespace2))$
1237 # A cluster-scoped kind has no namespace: filter it by a path, such as object.metadata.name
1238
1239 [input.kubernetes_watch::nodes]
1240 disabled = false
1241
1242 # How often every object is forwarded again; one no longer listed is forwarded as DELETED
1243 # with "reason":"absent_from_relist"
1244 refresh = 45m
1245
1246 apiVersion = v1
1247 kind = Node
1248 namespace =
1249
1250 type = objects
1251
1252 elasticsearch.datastream =
1253
1254 output =
1255
1256 excludeManagedFields = true
1257
1258 # you can remove or hash some values in the events (after modifyValues you can define path in the JSON object,
1259 # and the value can be hash:{hashFunction}, or remove to remove the object )
1260 ; modifyValues.object.data.* = hash:sha256
1261 ; modifyValues.object.metadata.annotations.* = remove
1262
1263 # You can exclude events by namespace with blacklist or whitelist only required namespaces
1264 # blacklist.kubernetes.namespace.name = ^namespace0$
1265 # whitelist.kubernetes.namespace.name = ^((namespace1)|(namespace2))$
1266 # A cluster-scoped kind has no namespace: filter it by a path, such as object.metadata.name
1267
1268 [input.kubernetes_watch::pods]
1269 disabled = false
1270
1271 # How often every object is forwarded again; one no longer listed is forwarded as DELETED
1272 # with "reason":"absent_from_relist"
1273 refresh = 45m
1274
1275 apiVersion = v1
1276 kind = Pod
1277 namespace =
1278
1279 type = objects
1280
1281 elasticsearch.datastream =
1282
1283 output =
1284
1285 excludeManagedFields = true
1286
1287 # you can remove or hash some values in the events (after modifyValues you can define path in the JSON object,
1288 # and the value can be hash:{hashFunction}, or remove to remove the object )
1289 ; modifyValues.object.data.* = hash:sha256
1290 ; modifyValues.object.metadata.annotations.* = remove
1291
1292 # You can exclude events by namespace with blacklist or whitelist only required namespaces
1293 # blacklist.kubernetes.namespace.name = ^namespace0$
1294 # whitelist.kubernetes.namespace.name = ^((namespace1)|(namespace2))$
1295 # A cluster-scoped kind has no namespace: filter it by a path, such as object.metadata.name
1296
1297 [input.kubernetes_watch::resourcequota]
1298 disabled = false
1299
1300 # How often every object is forwarded again; one no longer listed is forwarded as DELETED
1301 # with "reason":"absent_from_relist"
1302 refresh = 45m
1303
1304 apiVersion = v1
1305 kind = ResourceQuota
1306 namespace =
1307
1308 type = objects
1309
1310 elasticsearch.datastream =
1311
1312 output =
1313
1314 excludeManagedFields = true
1315
1316 # you can remove or hash some values in the events (after modifyValues you can define path in the JSON object,
1317 # and the value can be hash:{hashFunction}, or remove to remove the object )
1318 ; modifyValues.object.data.* = hash:sha256
1319 ; modifyValues.object.metadata.annotations.* = remove
1320
1321 # You can exclude events by namespace with blacklist or whitelist only required namespaces
1322 # blacklist.kubernetes.namespace.name = ^namespace0$
1323 # whitelist.kubernetes.namespace.name = ^((namespace1)|(namespace2))$
1324 # A cluster-scoped kind has no namespace: filter it by a path, such as object.metadata.name1325 es-default-index-template.json: |
1326 {
1327 "version": 1,
1328 "data_stream": {},
1329 "_meta": {
1330 "description": "Default template for Collectord indexes"
1331 },
1332 "priority": 500,
1333 "template": {
1334 "settings": {
1335 "index": {
1336 "refresh_interval": "5s",
1337 "mapping": {
1338 "total_fields": {
1339 "limit": "10000"
1340 }
1341 },
1342 "max_docvalue_fields_search": "200"
1343 },
1344 "query": {
1345 "default_field": [
1346 "message",
1347 "container.name",
1348 "container.image.name",
1349 "host.name",
1350 "orchestrator.namespace",
1351 "orchestrator.cluster.name",
1352 "container.name",
1353 "kubernetes.pod.name",
1354 "kubernetes.pod.id",
1355 "kubernetes.pod.ip",
1356 "kubernetes.namespace.name",
1357 "kubernetes.node.name",
1358 "kubernetes.cluster.name",
1359 "kubernetes.replicaset.name",
1360 "kubernetes.deployment.name",
1361 "kubernetes.statefulset.name",
1362 "kubernetes.daemonset.name",
1363 "kubernetes.job.name",
1364 "kubernetes.cronjob.name"
1365 ]
1366 }
1367 },
1368 "mappings": {
1369 "dynamic_templates": [
1370 {
1371 "kubernetes.pod.labels.*": {
1372 "path_match": "kubernetes.pod.labels.*",
1373 "mapping": { "type": "keyword" },
1374 "match_mapping_type": "*"
1375 }
1376 },
1377 {
1378 "kubernetes.namespace.labels.*": {
1379 "path_match": "kubernetes.namespace.labels.*",
1380 "mapping": { "type": "keyword" },
1381 "match_mapping_type": "*"
1382 }
1383 },
1384 {
1385 "kubernetes.node.labels.*": {
1386 "path_match": "kubernetes.node.labels.*",
1387 "mapping": { "type": "keyword" },
1388 "match_mapping_type": "*"
1389 }
1390 },
1391 {
1392 "kubernetes.daemonset.labels.*": {
1393 "path_match": "kubernetes.daemonset.labels.*",
1394 "mapping": { "type": "keyword" },
1395 "match_mapping_type": "*"
1396 }
1397 },
1398 {
1399 "kubernetes.replicaset.labels.*": {
1400 "path_match": "kubernetes.replicaset.labels.*",
1401 "mapping": { "type": "keyword" },
1402 "match_mapping_type": "*"
1403 }
1404 },
1405 {
1406 "kubernetes.deployment.labels.*": {
1407 "path_match": "kubernetes.deployment.labels.*",
1408 "mapping": { "type": "keyword" },
1409 "match_mapping_type": "*"
1410 }
1411 },
1412 {
1413 "kubernetes.statefulset.labels.*": {
1414 "path_match": "kubernetes.statefulset.labels.*",
1415 "mapping": { "type": "keyword" },
1416 "match_mapping_type": "*"
1417 }
1418 },
1419 {
1420 "kubernetes.job.labels.*": {
1421 "path_match": "kubernetes.job.labels.*",
1422 "mapping": { "type": "keyword" },
1423 "match_mapping_type": "*"
1424 }
1425 },
1426 {
1427 "kubernetes.cronjob.labels.*": {
1428 "path_match": "kubernetes.cronjob.labels.*",
1429 "mapping": { "type": "keyword" },
1430 "match_mapping_type": "*"
1431 }
1432 },
1433 {
1434 "strings_as_keyword": {
1435 "mapping": { "ignore_above": 1024, "type": "keyword" },
1436 "match_mapping_type": "string"
1437 }
1438 }
1439 ],
1440 "properties": {
1441 "@timestamp": { "type": "date"},
1442 "message": {"type":"text"},
1443 "collectord_errors": {"type": "keyword","ignore_above": 1024},
1444 "collectord_repeat_count": {"type": "long"},
1445 "collectord_repeat_span": {"type": "long"},
1446 "collectord_repeat_offsets": {"type": "long"},
1447 "collectord_fold_count": {"type": "long"},
1448 "collectord_fold_bytes": {"type": "long"},
1449 "collectord_pattern_lines": {"type": "long"},
1450 "collectord_pattern_bytes": {"type": "long"},
1451 "collectord_pattern_seconds": {"type": "long"},
1452 "collectord": {"properties": {
1453 "repeat": {"properties": {
1454 "count": {"type": "long"},
1455 "span": {"type": "long"},
1456 "offsets": {"type": "long"}
1457 }},
1458 "fold": {"properties": {
1459 "count": {"type": "long"},
1460 "bytes": {"type": "long"}
1461 }},
1462 "pattern": {"properties": {
1463 "lines": {"type": "long"},
1464 "bytes": {"type": "long"},
1465 "seconds": {"type": "long"}
1466 }}
1467 }},
1468 "input":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},
1469 "container": {"properties": {
1470 "name": { "ignore_above": 1024, "type": "keyword"},
1471 "id": { "ignore_above": 1024, "type": "keyword"},
1472 "runtime": {"ignore_above": 1024, "type": "keyword"},
1473 "image": {"properties": {
1474 "name": { "ignore_above": 1024, "type": "keyword"}
1475 }}
1476 }
1477 },
1478 "event": {"properties": {
1479 "id": {"ignore_above": 1024, "type": "keyword"}
1480 }},
1481 "host": {"properties": {
1482 "name": {"ignore_above": 1024, "type": "keyword"},
1483 "architecture": {"ignore_above": 1024, "type": "keyword"},
1484 "hostname": {"ignore_above": 1024, "type": "keyword"}
1485 }},
1486 "log": { "properties": {
1487 "file": { "properties": {
1488 "path": {"ignore_above": 1024, "type": "keyword"}
1489 }},
1490 "offset": {"type": "long"},
1491 "syslog": {"properties": {
1492 "appname": {"ignore_above": 1024, "type": "keyword"},
1493 "facility": {"properties": {
1494 "code": {"type": "long"}
1495 }},
1496 "priority": {"type": "long"},
1497 "procid": {"type": "long"},
1498 "hostname": {"ignore_above": 1024, "type": "keyword"}
1499 }}
1500 }},
1501 "orchestrator": {"properties": {
1502 "cluster": {"properties": {
1503 "name": {"ignore_above": 1024, "type": "keyword"}
1504 }},
1505 "namespace": {"properties": {
1506 "name":{"path":"kubernetes.namespace.name","type":"alias"}
1507 }},
1508 "type": {"ignore_above": 1024, "type": "keyword"}
1509 }},
1510 "stream": {"ignore_above": 1024, "type": "keyword"},
1511 "agent":{"properties":{
1512 "hostname":{"path":"agent.name","type":"alias"},
1513 "name":{"ignore_above":1024,"type":"keyword"},
1514 "id":{"ignore_above":1024,"type":"keyword"},
1515 "type":{"ignore_above":1024,"type":"keyword"},
1516 "ephemeral_id":{"ignore_above":1024,"type":"keyword"},
1517 "version":{"ignore_above":1024,"type":"keyword"}
1518 }},
1519 "ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},
1520 "kubernetes": { "properties": {
1521 "container": {"properties": {
1522 "imageid": {"ignore_above": 1024, "type": "keyword"}
1523 }},
1524 "host": {"properties": {
1525 "ip": {"type": "ip"}
1526 }},
1527 "namespace": {"properties": {
1528 "name": {"ignore_above": 1024, "type": "keyword"}
1529 }},
1530 "node": { "properties": {
1531 "id": {"ignore_above": 1024, "type": "keyword"},
1532 "name": {"ignore_above": 1024,"type": "keyword"}
1533 }},
1534 "pod": {"properties": {
1535 "id": {"ignore_above": 1024,"type": "keyword"},
1536 "ip": {"type": "ip"},
1537 "name": {"ignore_above": 1024,"type": "keyword"}
1538 }},
1539 "statefulset": { "properties": {
1540 "id": {"ignore_above": 1024,"type": "keyword"},
1541 "name": {"ignore_above": 1024,"type": "keyword"}
1542 }},
1543 "replicaset": {"properties": {
1544 "id": {"ignore_above": 1024,"type": "keyword"},
1545 "name": {"ignore_above": 1024, "type": "keyword"}
1546 }},
1547 "deployment": { "properties": {
1548 "id": {"ignore_above": 1024,"type": "keyword"},
1549 "name": {"ignore_above": 1024,"type": "keyword" }
1550 }},
1551 "cronjob": {"properties": {
1552 "id": {"ignore_above": 1024,"type": "keyword"},
1553 "name": {"ignore_above": 1024, "type": "keyword"}
1554 }},
1555 "job": {"properties": {
1556 "id": {"ignore_above": 1024,"type": "keyword"},
1557 "name": {"ignore_above": 1024, "type": "keyword"}
1558 }}
1559 }},
1560 "volume": {"properties": {
1561 "name": {"ignore_above": 1024,"type": "keyword"}
1562 }}
1563 }
1564 }
1565 },
1566 "index_patterns": ["logs-collectord-${COLLECTORD_VERSION}"]
1567 }
1568
1569 es-failed-index-template.json: |
1570 {
1571 "version": 1,
1572 "data_stream": {},
1573 "_meta": {
1574 "description": "Default template for Collectord indexes for events that failed to be ingested"
1575 },
1576 "priority": 1000,
1577 "template": {
1578 "settings": {
1579 "index": {
1580 "refresh_interval": "5s",
1581 "mapping": {
1582 "total_fields": {
1583 "limit": "10000"
1584 }
1585 },
1586 "max_docvalue_fields_search": "200"
1587 },
1588 "query": {
1589 "default_field": [
1590 "message"
1591 ]
1592 }
1593 },
1594 "mappings": {
1595 "properties": {
1596 "@timestamp": {
1597 "type": "date"},
1598 "message": {
1599 "type": "text"}
1600 }
1601 }
1602 },
1603 "index_patterns": [
1604 "logs-collectord-failed-${COLLECTORD_VERSION}"]
1605 }
1606---
1607apiVersion: apps/v1
1608kind: DaemonSet
1609metadata:
1610 name: collectorforkubernetes-elasticsearch
1611 namespace: collectorforkubernetes
1612 annotations:
1613 collectord.collectord.io/logs-replace.ignoredebuglog-search: ^(TRACE|DEBUG) .*
1614 collectord.collectord.io/logs-replace.ignoredebuglog-val: ""
1615 labels:
1616 app: collectorforkubernetes-elasticsearch
1617spec:
1618 updateStrategy:
1619 type: RollingUpdate
1620 selector:
1621 matchLabels:
1622 daemon: collectorforkubernetes-elasticsearch
1623 template:
1624 metadata:
1625 name: collectorforkubernetes-elasticsearch
1626 annotations:
1627 # A change to the configuration files these pods mount rolls them:
1628 # Collectord reads its inputs' configuration at startup. A ConfigMap managed
1629 # outside the chart (configMap.create: false) gets no checksum: roll the pods
1630 # with an annotation of your own under podAnnotations.
1631 checksum/config: d8508fe54b1442555bc52064b6627fbce02fc14fc7abb4737816d2c7ca64a10e
1632 labels:
1633 daemon: collectorforkubernetes-elasticsearch
1634 spec:
1635 priorityClassName: "system-node-critical"
1636 dnsPolicy: ClusterFirstWithHostNet
1637 hostNetwork: true
1638 serviceAccountName: collectorforkubernetes
1639 nodeSelector:
1640 kubernetes.io/os: linux
1641 affinity:
1642 nodeAffinity:
1643 requiredDuringSchedulingIgnoredDuringExecution:
1644 nodeSelectorTerms:
1645 - matchExpressions:
1646 - key: eks.amazonaws.com/compute-type
1647 operator: NotIn
1648 values:
1649 - fargate
1650 - key: type
1651 operator: NotIn
1652 values:
1653 - virtual-kubelet
1654 tolerations:
1655 - effect: NoSchedule
1656 operator: Exists
1657 - effect: NoExecute
1658 operator: Exists
1659 containers:
1660 - name: collectorforkubernetes
1661 image: registry.outcold.solutions/outcoldsolutions/collectorforkubernetes:26.10.1
1662 imagePullPolicy: Always
1663 livenessProbe:
1664 failureThreshold: 3
1665 httpGet:
1666 host: 127.0.0.1
1667 path: /healthz
1668 port: 11892
1669 initialDelaySeconds: 120
1670 periodSeconds: 60
1671 timeoutSeconds: 5
1672 readinessProbe:
1673 failureThreshold: 3
1674 httpGet:
1675 host: 127.0.0.1
1676 path: /readyz
1677 port: 11892
1678 initialDelaySeconds: 10
1679 periodSeconds: 15
1680 timeoutSeconds: 5
1681 securityContext:
1682 runAsUser: 0
1683 privileged: true
1684 resources:
1685 limits:
1686 cpu: 2000m
1687 memory: 1024Mi
1688 requests:
1689 cpu: 500m
1690 memory: 256Mi
1691 env:
1692 - name: KUBERNETES_NODENAME
1693 valueFrom:
1694 fieldRef:
1695 fieldPath: spec.nodeName
1696 - name: KUBERNETES_NODEIP
1697 valueFrom:
1698 fieldRef:
1699 fieldPath: status.hostIP
1700 - name: POD_NAME
1701 valueFrom:
1702 fieldRef:
1703 fieldPath: metadata.name
1704 volumeMounts:
1705 - name: collectorforkubernetes-state
1706 mountPath: /data
1707 - name: collectorforkubernetes-config
1708 mountPath: /config/
1709 readOnly: true
1710 - name: rootfs
1711 mountPath: /rootfs/
1712 readOnly: false
1713 mountPropagation: HostToContainer
1714 - name: localtime
1715 mountPath: /etc/localtime
1716 readOnly: true
1717
1718 volumes:
1719 - name: collectorforkubernetes-state
1720 hostPath:
1721 path: /var/lib/collectorforkubernetes-elasticsearch/data/
1722 type: DirectoryOrCreate
1723 - name: rootfs
1724 hostPath:
1725 path: /
1726 - name: localtime
1727 hostPath:
1728 path: /etc/localtime
1729 - name: collectorforkubernetes-config
1730 configMap:
1731 name: collectorforkubernetes-elasticsearch
1732 items:
1733 - key: 001-general.conf
1734 path: 001-general.conf
1735 - key: 002-daemonset.conf
1736 path: 002-daemonset.conf
1737 - key: es-default-index-template.json
1738 path: es-default-index-template.json
1739 - key: es-failed-index-template.json
1740 path: es-failed-index-template.json
1741---
1742apiVersion: apps/v1
1743kind: Deployment
1744metadata:
1745 name: collectorforkubernetes-elasticsearch-addon
1746 namespace: collectorforkubernetes
1747 annotations:
1748 collectord.collectord.io/logs-replace.ignoredebuglog-search: ^(TRACE|DEBUG) .*
1749 collectord.collectord.io/logs-replace.ignoredebuglog-val: ""
1750 labels:
1751 app: collectorforkubernetes-elasticsearch
1752spec:
1753 replicas: 1
1754 selector:
1755 matchLabels:
1756 daemon: collectorforkubernetes-elasticsearch
1757 template:
1758 metadata:
1759 name: collectorforkubernetes-elasticsearch-addon
1760 annotations:
1761 # A change to the configuration files these pods mount rolls them:
1762 # Collectord reads its inputs' configuration at startup. A ConfigMap managed
1763 # outside the chart (configMap.create: false) gets no checksum: roll the pods
1764 # with an annotation of your own under podAnnotations.
1765 checksum/config: 7a25544e17975af6d1162b003e28d25e3b0eec9aacc91688bb2275e2c2fb3b50
1766 labels:
1767 daemon: collectorforkubernetes-elasticsearch
1768 spec:
1769 priorityClassName: "system-cluster-critical"
1770 serviceAccountName: collectorforkubernetes
1771 nodeSelector:
1772 kubernetes.io/os: linux
1773 affinity:
1774 nodeAffinity:
1775 requiredDuringSchedulingIgnoredDuringExecution:
1776 nodeSelectorTerms:
1777 - matchExpressions:
1778 - key: eks.amazonaws.com/compute-type
1779 operator: NotIn
1780 values:
1781 - fargate
1782 - key: type
1783 operator: NotIn
1784 values:
1785 - virtual-kubelet
1786 containers:
1787 - name: collectorforkubernetes
1788 image: registry.outcold.solutions/outcoldsolutions/collectorforkubernetes:26.10.1
1789 imagePullPolicy: Always
1790 livenessProbe:
1791 failureThreshold: 3
1792 httpGet:
1793 path: /healthz
1794 port: 11892
1795 initialDelaySeconds: 120
1796 periodSeconds: 60
1797 timeoutSeconds: 5
1798 readinessProbe:
1799 failureThreshold: 3
1800 httpGet:
1801 path: /readyz
1802 port: 11892
1803 initialDelaySeconds: 10
1804 periodSeconds: 15
1805 timeoutSeconds: 5
1806 securityContext:
1807 allowPrivilegeEscalation: false
1808 capabilities:
1809 drop:
1810 - ALL
1811 privileged: false
1812 runAsUser: 0
1813 seLinuxOptions:
1814 type: spc_t
1815 seccompProfile:
1816 type: RuntimeDefault
1817 resources:
1818 limits:
1819 cpu: 1000m
1820 memory: 512Mi
1821 requests:
1822 cpu: 200m
1823 memory: 64Mi
1824 env:
1825 - name: KUBERNETES_NODENAME
1826 valueFrom:
1827 fieldRef:
1828 fieldPath: spec.nodeName
1829 - name: POD_NAME
1830 valueFrom:
1831 fieldRef:
1832 fieldPath: metadata.name
1833 volumeMounts:
1834 - name: collectorforkubernetes-state
1835 mountPath: /data
1836 - name: collectorforkubernetes-config
1837 mountPath: /config/
1838 readOnly: true
1839 - name: localtime
1840 mountPath: /etc/localtime
1841 readOnly: true
1842
1843 volumes:
1844 # The DaemonSet pod on this node owns the directory and the database the
1845 # addon opens read-only. DirectoryOrCreate, so an addon scheduled first
1846 # mounts it too; Collectord waits for the DaemonSet to create the database.
1847 - name: collectorforkubernetes-state
1848 hostPath:
1849 path: /var/lib/collectorforkubernetes-elasticsearch/data/
1850 type: DirectoryOrCreate
1851 - name: localtime
1852 hostPath:
1853 path: /etc/localtime
1854 - name: collectorforkubernetes-config
1855 configMap:
1856 name: collectorforkubernetes-elasticsearch
1857 items:
1858 - key: 001-general.conf
1859 path: 001-general.conf
1860 - key: 004-addon.conf
1861 path: 004-addon.conf
1862 - key: es-default-index-template.json
1863 path: es-default-index-template.json
1864 - key: es-failed-index-template.json
1865 path: es-failed-index-template.json