Download
Link
collectorforopenshift-rhel-syslog.yaml
CURL
1curl -O https://www.outcoldsolutions.com/docs/syslog-kubernetes/collectorforopenshift-rhel-syslog.yamlWGET
1wget https://www.outcoldsolutions.com/docs/syslog-kubernetes/collectorforopenshift-rhel-syslog.yamlcollectorforopenshift-rhel-syslog.yaml
1apiVersion: project.openshift.io/v1
2kind: Project
3metadata:
4 labels:
5 app: collectorforopenshift-syslog
6 name: collectorforopenshift-syslog
7 annotations:
8 openshift.io/node-selector: ''
9 openshift.io/description: 'Forwarding logs to Syslog, built by Outcold Solutions'
10 openshift.io/display-name: 'Collectord for OpenShift (Syslog)'
11---
12apiVersion: apiextensions.k8s.io/v1
13kind: CustomResourceDefinition
14metadata:
15 name: configurations.collectord.io
16spec:
17 group: collectord.io
18 versions:
19 - name: v1
20 served: true
21 storage: true
22 schema:
23 openAPIV3Schema:
24 type: object
25 properties:
26 spec:
27 type: object
28 additionalProperties: true
29 force:
30 type: boolean
31 scope: Cluster
32 names:
33 listKind: ConfigurationList
34 plural: configurations
35 singular: configuration
36 kind: Configuration
37---
38apiVersion: apiextensions.k8s.io/v1
39kind: CustomResourceDefinition
40metadata:
41 name: splunkoutputs.collectord.io
42spec:
43 group: collectord.io
44 versions:
45 - name: v1
46 served: true
47 storage: true
48 schema:
49 openAPIV3Schema:
50 type: object
51 properties:
52 spec:
53 type: object
54 properties:
55 url:
56 type: string
57 format: uri
58 insecure:
59 type: boolean
60 token:
61 type: string
62 description: "Plain token"
63 tokenFromSecret:
64 type: object
65 description: "Reference to a Kubernetes Secret"
66 properties:
67 secret:
68 type: string
69 key:
70 type: string
71 defaultMetricsIndex:
72 type: string
73 description: "Index of the metrics-index records sent through this output; empty for the token's default"
74 incorrectIndexBehavior:
75 type: string
76 description: "What happens to a record the token refuses"
77 enum: ["RedirectToDefault", "Drop", "Retry"]
78 requireExplicitIndex:
79 type: boolean
80 description: "Drop a record that carries no index instead of sending it to the token's default"
81 oneOf:
82 - required: ["token"]
83 - required: ["tokenFromSecret"]
84 scope: Namespaced
85 names:
86 listKind: SplunkOutputList
87 plural: splunkoutputs
88 singular: splunkoutput
89 kind: SplunkOutput
90---
91apiVersion: v1
92kind: ServiceAccount
93metadata:
94 labels:
95 app: collectorforopenshift-syslog
96 name: collectorforopenshift-syslog
97 namespace: collectorforopenshift-syslog
98---
99kind: SecurityContextConstraints
100apiVersion: security.openshift.io/v1
101metadata:
102 name: collectorforopenshift-syslog
103 annotations:
104 "helm.sh/hook": pre-install,pre-upgrade
105 "helm.sh/hook-weight": "-10"
106 "helm.sh/hook-delete-policy": before-hook-creation
107allowHostDirVolumePlugin: true
108allowHostIPC: true
109allowHostNetwork: true
110allowHostPID: true
111allowHostPorts: true
112allowPrivilegeEscalation: true
113allowPrivilegedContainer: true
114readOnlyRootFilesystem: false
115allowedCapabilities:
116 - '*'
117allowedUnsafeSysctls:
118 - '*'
119fsGroup:
120 type: RunAsAny
121runAsUser:
122 type: RunAsAny
123seLinuxContext:
124 type: RunAsAny
125supplementalGroups:
126 type: RunAsAny
127seccompProfiles:
128 - '*'
129users:
130 - system:serviceaccount:collectorforopenshift-syslog:collectorforopenshift-syslog
131volumes:
132 - '*'
133---
134apiVersion: rbac.authorization.k8s.io/v1
135kind: ClusterRole
136metadata:
137 name: collectorforopenshift-syslog
138 labels:
139 app: collectorforopenshift-syslog
140rules:
141 - apiGroups:
142 - ""
143 resources:
144 - events
145 - namespaces
146 - nodes
147 - nodes/metrics
148 - persistentvolumeclaims
149 - persistentvolumes
150 - pods
151 - replicationcontrollers
152 - resourcequotas
153 - services
154 verbs:
155 - get
156 - list
157 - watch
158 - apiGroups:
159 - apps
160 resources:
161 - daemonsets
162 - deployments
163 - replicasets
164 - statefulsets
165 verbs:
166 - get
167 - list
168 - watch
169 - apiGroups:
170 - batch
171 resources:
172 - cronjobs
173 - jobs
174 verbs:
175 - get
176 - list
177 - watch
178 - apiGroups:
179 - discovery.k8s.io
180 resources:
181 - endpointslices
182 verbs:
183 - get
184 - list
185 - watch
186 - apiGroups:
187 - rbac.authorization.k8s.io
188 resources:
189 - clusterroles
190 verbs:
191 - get
192 - list
193 - watch
194 - apiGroups:
195 - collectord.io
196 resources:
197 - configurations
198 - splunkoutputs
199 verbs:
200 - get
201 - list
202 - watch
203 - apiGroups: []
204 nonResourceURLs:
205 - /metrics
206 resources: []
207 verbs:
208 - get
209 - apiGroups: []
210 nonResourceURLs:
211 - /api
212 - /api/*
213 - /apis
214 - /apis/*
215 resources: []
216 verbs:
217 - get
218---
219apiVersion: rbac.authorization.k8s.io/v1
220kind: ClusterRoleBinding
221metadata:
222 name: collectorforopenshift-syslog
223 labels:
224 app: collectorforopenshift-syslog
225roleRef:
226 apiGroup: rbac.authorization.k8s.io
227 kind: ClusterRole
228 name: collectorforopenshift-syslog
229subjects:
230 - kind: ServiceAccount
231 name: collectorforopenshift-syslog
232 namespace: collectorforopenshift-syslog
233---
234apiVersion: v1
235kind: ResourceQuota
236metadata:
237 name: collectorforopenshift-syslog-critical-pods
238 namespace: collectorforopenshift-syslog
239 labels:
240 app: collectorforopenshift-syslog
241spec:
242 hard:
243 pods: 1G
244 scopeSelector:
245 matchExpressions:
246 - scopeName: PriorityClass
247 operator: In
248 values:
249 - system-cluster-critical
250 - system-node-critical
251---
252apiVersion: v1
253kind: ConfigMap
254metadata:
255 name: collectorforopenshift-syslog
256 namespace: collectorforopenshift-syslog
257 labels:
258 app: collectorforopenshift-syslog
259data:
260 001-general.conf: |261 # The general configuration is used for all deployments
262 #
263 # Run collectord with the flag -conf and specify location of the configuration files.
264 #
265 # You can override all the values using environment variables with the format like
266 # COLLECTOR__<ANYNAME>=<section>__<key>=<value>
267 # As an example you can set dataPath in [general] section as
268 # COLLECTOR__DATAPATH=general__dataPath=C:\\some\\path\\data.db
269 # This parameter can be configured using -env-override, set it to empty string to disable this feature
270
271 [general]
272
273 # Please review license https://www.outcoldsolutions.com/legal/license-agreement/
274 # and accept license by changing the value to *true*
275 acceptLicense = false
276
277 # Location for the database
278 # Collectord stores positions of the files and internal state
279 dataPath = ./data/
280
281 # log level (accepted values are trace, debug, info, warn, error, fatal)
282 logLevel = info
283
284 # Internal http server - serves the liveness (/healthz) and readiness (/readyz)
285 # probes. DaemonSets run on the host network, so bind to loopback (127.0.0.1:11888)
286 # to keep the endpoints off the node IP; the kubelet still reaches them via
287 # httpGet.host 127.0.0.1. The addon runs off the host network and overrides the
288 # bind to 0.0.0.0 in 004-addon.conf so the kubelet can reach it on the pod IP.
289 # httpServerBinding = 0.0.0.0:11888
290 httpServerBinding =
291
292 httpServerLog = false
293
294 # telemetry report endpoint, set it to empty string to disable telemetry
295 telemetryEndpoint = https://license.outcold.solutions/telemetry/
296
297 licenseEndpoint = https://license.outcold.solutions/license/
298
299 # Only for the Outcold Solutions License Server; one on-premises uses [license.client]
300 licenseServerProxyUrl =
301
302 # user:password
303 licenseServerProxyBasicAuth =
304
305 license =
306
307 # Environment variable $KUBERNETES_NODENAME is used by default to setup hostname
308 # Use value below to override specific name
309 hostname =
310
311 # Default output for events, logs and metrics
312 # valid values: splunk and devnull
313 # Use devnull by default if you don't want to redirect data
314 defaultOutput = syslog
315
316 fileInputBufferSize = 8kb
317
318 fileInputLineMaxSize = 1mb
319
320 # Include custom fields to attach to every event, in example below every event sent to Splunk will hav
321 # indexed field my_environment=dev. Fields names should match to ^[a-z][_a-z0-9]*$
322 # Better way to configure that is to specify labels for Kubernetes Nodes.
323 # ; fields.my_environment = dev
324 # Identify the cluster if you are planning to monitor multiple clusters
325 fields.cluster = -
326
327 # Include EC2 Metadata (see list of possible fields https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
328 # Should be in format ec2Metadata.{desired_field_name} = {url path to read the value}
329 # ec2Metadata.ec2_instance_id = /latest/meta-data/instance-id
330 # ec2Metadata.ec2_instance_type = /latest/meta-data/instance-type
331
332 # subdomain for the annotations added to the pods, workloads, namespaces or containers, like splunk.collectord.io/..
333 annotationsSubdomain = syslog
334
335 # configure global thruput per second for forwarded logs (metrics are not included)
336 # for example if you set `thruputPerSecond = 512Kb`, that will limit amount of logs forwarded
337 # from the single Collectord instance to 512Kb per second.
338 # You can configure thruput individually for the logs (including specific for container logs) below
339 thruputPerSecond =
340
341 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
342 # older than 7 days
343 tooOldEvents =
344
345 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
346 tooNewEvents =
347
348 # How Collectord spells the fields it composes: "_" (openshift_pod_name, the default) or "."
349 fieldsSeparator =
350
351 # For input.files::X and application logs, when glob or match are configured, Collectord can automatically
352 # detect gzipped files and skip them (based on the extensions or magic numbers)
353 autoSkipGzipFiles = true
354
355 # Publish to non-default outputs asynchronously (default), so a slow or down output does not
356 # block the others; false holds the pipeline back instead of dropping events.
357 multioutput.async = true
358
359 # Buffer size for the async proxy (default 100), per output of each pipeline. When it and
360 # the output's queue are full, events are dropped (multioutput_dropped_events).
361 multioutput.asyncBufferSize = 100
362
363 [license.client]
364 # point to the license located on the HTTP web server, or a hosted by the Collectord running as license server
365 url =
366 basicAuth =
367 insecure = false
368 capath =
369 caname =
370 proxyUrl =
371
372 # user:password
373 proxyBasicAuth =
374
375 [general.kubernetes]
376
377 # Override service URL for Kubernetes (default is ${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT})
378 serviceURL =
379
380 # Environment variable $KUBERNETES_NODENAME is used by default to setup nodeName
381 # Use it only when you need to override it
382 nodeName =
383
384 tokenPath = /var/run/secrets/kubernetes.io/serviceaccount/token
385 certPath = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
386
387 # Default timeout for http responses. The streaming/watch requests depend on this timeout.
388 timeout = 30m
389
390 # How long to keep the cache for the recent calls to API server (to limit number of calls when collectord discovers new pods)
391 metadataTTL = 30s
392
393 podsCgroupFilter = ^/([^/\s]+/)*kubepods(\.slice)?/((kubepods-)?(burstable|besteffort)(\.slice)?/)?([^/]*)pod([0-9a-f]{32}|[0-9a-f\-_]{36})(\.slice)?$
394
395 containersCgroupFilter = ^/([^/\s]+/)*kubepods(\.slice)?/((kubepods-)?(burstable|besteffort)(\.slice)?/)?([^/]*)pod([0-9a-f]{32}|[0-9a-f\-_]{36})(\.slice)?/(docker-|crio-|cri-\w+-)?[0-9a-f]{64}(\.scope)?(\/.+)?$
396
397 # path to the kubelet root location (use it to discover application logs for emptyDir)
398 # the expected format is `pods/{pod-id}/volumes/kubernetes.io~empty-dir/{volume-name}/_data/`
399 volumesRootDir = /rootfs/var/lib/kubelet/
400
401 # You can attach annotations as a metadata, using the format
402 # includeAnnotations.{key} = {regexp}
403 # For example if you want to include all annotations that starts with `prometheus.io` or `example.com` you can include
404 # the following format:
405 # includeAnnotations.1 = ^prometheus\.io.*
406 # includeAnnotations.2 = ^example\.com.*
407
408 # You can exclude labels from metadata, using the format
409 # excludeLabels.{key} = {regexp}
410 # For example if you want to exclude all labels that starts with `prometheus.io` or `example.com` you can include
411 # the following format:
412 # excludeLabels.1 = ^prometheus\.io.*
413 # excludeLabels.2 = ^example\.com.*
414
415 # watch for changes (annotations) in the objects
416 watch.namespace = v1/namespace
417 watch.deployment = apps/v1/deployment
418 watch.configuration = collectord.io/v1/configuration
419
420 # Collectord can review the assigned ClusterRole and traverse metadata for the Pods only for the Owner objects
421 # that are defined in the ClusterRole, ignoring anything else, it does not have access to.
422 # This way Collectord does not generate 403 requests on API Server
423 clusterRole = collectorforopenshift-syslog
424
425 # Alternative of telling Collectord about the ClusterRole is to manually list the objects.
426 # You can define which objects Collectord should traverse when it sees Owners.
427 ; traverseOwnership.namespaces = v1/namespace
428
429 [output.syslog]
430
431 # tcp or udp
432 network = tcp
433 address =434 002-daemonset.conf: | 435 [general.cri-o]
436
437 # url for CRIO API, only unix socket is supported
438 url = unix:///rootfs/var/run/crio/crio.sock
439
440 # Timeout for http responses to docker client. The streaming requests depend on this timeout.
441 timeout = 1m
442
443 [general.containerd]
444 # Runtime can be on /rootfs/run/containerd (depends on the Linux distribution)
445 runtimePath = /rootfs/var/run/containerd
446 namespace = k8s.io
447
448 [input.files]
449
450 disabled = false
451
452 # root location of docker log files
453 # logs are expected in standard docker format like {containerID}/{containerID}-json.log
454 # rotated files
455 path = /rootfs/var/lib/docker/containers/
456
457 # root location of CRI-O files
458 # logs are expected in Kubernetes format, like {podID}/{containerName}/0.log
459 crioPath = /rootfs/var/log/pods/
460
461 # files are read using polling schema, when reach the EOF how often to check if files got updated
462 pollingInterval = 250ms
463
464 # how often to look for the new files under logs path
465 walkingInterval = 5s
466
467 # include verbose fields in events (file offset)
468 verboseFields = false
469
470 type = k8s_logs
471
472 syslog.format = {{type}}|{{timestamp::format(2006-01-02T15:04:05.999999999Z07:00)}}|{{cluster}}|{{host}}|{{namespace}}|{{pod_id}}|{{pod_name}}|{{container_name}}|{{stream}}|{{message}}
473
474 # docker splits events when they are larger than 10-100k (depends on the docker version)
475 # we join them together by default and forward to Splunk as one event
476 joinPartialEvents = true
477
478 # In case if your containers report messages with terminal colors or other escape sequences
479 # you can enable strip for all the containers in one place.
480 # Better is to enable it only for required container with the annotation syslog.collectord.io/logs-escapeterminalsequences=true
481 stripTerminalEscapeSequences = false
482
483 # Regexp used for stripping terminal colors, it does not stip all the escape sequences
484 # Read http://man7.org/linux/man-pages/man4/console_codes.4.html for more information
485 stripTerminalEscapeSequencesRegex = (\x1b\[\d{1,3}(;\d{1,3})*m)|(\x07)|(\x1b]\d+(\s\d)?;[^\x07]+\x07)|(.*\x1b\[K)
486
487 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
488 samplingPercent = -1
489
490 # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
491 samplingKey =
492
493 output =
494
495 # configure default thruput per second for for each container log
496 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
497 # from the single container to 128Kb per second.
498 thruputPerSecond =
499
500 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
501 # older than 7 days
502 tooOldEvents =
503
504 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
505 tooNewEvents =
506
507 # Read rotated logs the kubelet compressed before Collectord read them; false counts them lost
508 readCompressed = true
509
510 # Bytes of those logs inflated at once, node-wide, under [general] dataPath
511 compressedInflateBudget = 256MiB
512
513 # Group each stream's repeated lines into one event per run that counts them.
514 # Off unless true; a workload overrides it with its
515 # syslog.collectord.io/logs-repeat-enabled annotation.
516 repeatEnabled = false
517
518 # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
519 # repeatMaxSpan = 1m
520
521 # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
522 # repeatOffsets = false
523
524 # What each stream may remember: distinct lines, lines one event stands for, and bytes
525 # repeatMaxKeys = 256
526 # repeatMaxCount = 1000
527 # repeatMaxBytes = 256KB
528
529 # Fold each stream's lines of one pattern past foldMaxPerMinute into one summary a minute.
530 # Lossy: folded lines are not sent. Off unless true; a workload overrides it with its
531 # syslog.collectord.io/logs-fold-enabled annotation.
532 foldEnabled = false
533
534 # Lines of one pattern a minute sent before the rest fold
535 # foldMaxPerMinute = 60
536
537 # Lines never folded (a regular expression; empty: the default)
538 # foldKeep = (?i)error|exception|fatal|panic
539
540 # What each stream may keep a minute: its patterns and summaries
541 # foldMaxBytes = 256KB
542
543 # Report each stream's top patterns every interval, as events of their own.
544 # Off unless true; a workload overrides it with its
545 # syslog.collectord.io/logs-patterns-enabled annotation.
546 patternsEnabled = false
547
548 # How often a report goes out, from 1m to 24h
549 # patternsInterval = 15m
550
551 # How many patterns a report carries, up to 100
552 # patternsTop = 20
553
554 # What each stream's patterns may keep; past it a new one takes the place of the lightest
555 # patternsMaxBytes = 256KB
556
557 # The report events' type (empty: the input's type with _patterns after it)
558 # patternsType =
559
560 [input.app_logs]
561
562 disabled = false
563
564 # root location of mounts (applies to hostPath mounts only), if the hostPath differs inside container from the path on host
565 root = /rootfs/
566
567 # how often to review list of available volumes
568 syncInterval = 5s
569
570 glob = *.log*
571
572 # files are read using polling schema, when reach the EOF how often to check if files got updated
573 pollingInterval = 250ms
574
575 # how often to look for the new files under logs path
576 walkingInterval = 5s
577
578 # include verbose fields in events (file offset)
579 verboseFields = false
580
581 type = k8s_logs
582
583 syslog.format = {{type}}|{{timestamp::format(2006-01-02T15:04:05.999999999Z07:00)}}|{{cluster}}|{{host}}|{{namespace}}|{{pod_id}}|{{pod_name}}|{{container_name}}|{{file_name}}|{{message}}
584
585 # we split files using new line character, with this configuration you can specify what defines the new event
586 # after new line
587 eventPatternRegex = ^[^\s]
588 eventPatternMaxInterval = 100ms
589 eventPatternMaxWait = 1s
590 eventPatternMaxSize = 1MB
591
592 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
593 samplingPercent = -1
594
595 # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
596 samplingKey =
597
598 output =
599
600 # configure default thruput per second for for each container log
601 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
602 # from the single container to 128Kb per second.
603 thruputPerSecond =
604
605 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
606 # older than 7 days
607 tooOldEvents =
608
609 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
610 tooNewEvents =
611
612 # Configure how long Collectord should keep the file descriptors open for files, that has not been forwarded yet
613 # When using PVC, and if pipeline is lagging behind, Collectord holding open fd for files, can cause long termination
614 # of pods, as kubelet cannot unmount the PVC volume from the system
615 maxHoldAfterClose = 1800s
616
617 # Group each file's repeated lines into one event per run that counts them.
618 # Off unless true; a workload overrides it with its
619 # syslog.collectord.io/volume.<n>-logs-repeat-enabled annotation.
620 repeatEnabled = false
621
622 # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
623 # repeatMaxSpan = 1m
624
625 # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
626 # repeatOffsets = false
627
628 # What each file may remember: distinct lines, lines one event stands for, and bytes
629 # repeatMaxKeys = 256
630 # repeatMaxCount = 1000
631 # repeatMaxBytes = 256KB
632
633 # Fold each file's lines of one pattern past foldMaxPerMinute into one summary a minute.
634 # Lossy: folded lines are not sent. Off unless true; a workload overrides it with its
635 # syslog.collectord.io/volume.<n>-logs-fold-enabled annotation.
636 foldEnabled = false
637
638 # Lines of one pattern a minute sent before the rest fold
639 # foldMaxPerMinute = 60
640
641 # Lines never folded (a regular expression; empty: the default)
642 # foldKeep = (?i)error|exception|fatal|panic
643
644 # What each file may keep a minute: its patterns and summaries
645 # foldMaxBytes = 256KB
646
647 # Report each file's top patterns every interval, as events of their own.
648 # Off unless true; a workload overrides it with its
649 # syslog.collectord.io/volume.<n>-logs-patterns-enabled annotation.
650 patternsEnabled = false
651
652 # How often a report goes out, from 1m to 24h
653 # patternsInterval = 15m
654
655 # How many patterns a report carries, up to 100
656 # patternsTop = 20
657
658 # What each file's patterns may keep; past it a new one takes the place of the lightest
659 # patternsMaxBytes = 256KB
660
661 # The report events' type (empty: the input's type with _patterns after it)
662 # patternsType =
663
664 [input.files::audit-logs]
665
666 disabled = false
667
668 path = /rootfs/var/log/kube-apiserver/
669
670 glob = audit*.log
671
672 match =
673
674 # limit search only on one level
675 recursive = false
676 nodeSelector = node-role.kubernetes.io/master
677
678 # files are read using polling schema, when reach the EOF how often to check if files got updated
679 pollingInterval = 250ms
680
681 # how often o look for the new files under logs path
682 walkingInterval = 5s
683
684 # include verbose fields in events (file offset)
685 verboseFields = false
686
687 type = k8s_audit_logs
688
689 syslog.format = {{type}}|{{timestamp::format(2006-01-02T15:04:05.999999999Z07:00)}}|{{cluster}}|{{message}}
690
691 extraction = (?P<message>.*"stageTimestamp":"(?P<timestamp>[^"]+)".*)
692 extractionMessageField = message
693
694 timestampField = timestamp
695
696 # format for timestamp
697 # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006`
698 timestampFormat = 2006-01-02T15:04:05.999999999Z07:00
699
700 # timestamp location (if not defined by format)
701 timestampLocation =
702
703 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
704 samplingPercent = -1
705
706 # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
707 samplingKey =
708
709 output =
710
711 # configure default thruput per second for this files group
712 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
713 # from the files in this group to 128Kb per second.
714 thruputPerSecond =
715
716 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
717 # older than 7 days
718 tooOldEvents =
719
720 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
721 tooNewEvents =
722
723 # by default every new event should start from not space symbol
724 eventPattern =
725
726 # Blacklisting and whitelisting the logs
727 # whitelist.0 = ^regexp$
728 # blacklist.0 = ^regexp$
729
730 # Lines matching a thruputExempt pattern pass every thruput cap and sampling; with one set,
731 # a spent cap drops the other lines instead of pausing this input
732 # thruputExempt.0 = ^regexp$
733
734 # Group each file's repeated lines into one event per run that counts them.
735 # Off unless true.
736 repeatEnabled = false
737
738 # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
739 # repeatMaxSpan = 1m
740
741 # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
742 # repeatOffsets = false
743
744 # What each file may remember: distinct lines, lines one event stands for, and bytes
745 # repeatMaxKeys = 256
746 # repeatMaxCount = 1000
747 # repeatMaxBytes = 256KB
748
749 # Fold each file's lines of one pattern past foldMaxPerMinute into one summary a minute.
750 # Lossy: folded lines are not sent. Off unless true.
751 foldEnabled = false
752
753 # Lines of one pattern a minute sent before the rest fold
754 # foldMaxPerMinute = 60
755
756 # Lines never folded (a regular expression; empty: the default)
757 # foldKeep = (?i)error|exception|fatal|panic
758
759 # What each file may keep a minute: its patterns and summaries
760 # foldMaxBytes = 256KB
761
762 # Report each file's top patterns every interval, as events of their own.
763 # Off unless true.
764 patternsEnabled = false
765
766 # How often a report goes out, from 1m to 24h
767 # patternsInterval = 15m
768
769 # How many patterns a report carries, up to 100
770 # patternsTop = 20
771
772 # What each file's patterns may keep; past it a new one takes the place of the lightest
773 # patternsMaxBytes = 256KB
774
775 # The report events' type (empty: the input's type with _patterns after it)
776 # patternsType =
777
778 [input.files::logs]
779
780 disabled = false
781
782 path = /rootfs/var/log/
783
784 glob =
785
786 match = ^(([\w\-.]+\.log(.[\d\-]+)?)|(docker))$
787 exclude = ^(auth|cron|daemon|kern|lpr|mail|user|ufw)\.log(.[\d\-]+)?$
788
789 # files are read using polling schema, when reach the EOF how often to check if files got updated
790 pollingInterval = 250ms
791
792 # how often o look for the new files under logs path
793 walkingInterval = 5s
794
795 # include verbose fields in events (file offset)
796 verboseFields = false
797
798 type = k8s_host_logs
799
800 syslog.format = {{type}}|{{timestamp::format(2006-01-02T15:04:05.999999999Z07:00)}}|{{cluster}}|{{host}}|{{file_name}}|{{message}}
801
802 extraction =
803 extractionMessageField =
804
805 timestampField =
806
807 # format for timestamp
808 # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006`
809 timestampFormat =
810
811 # timestamp location (if not defined by format)
812 timestampLocation =
813
814 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
815 samplingPercent = -1
816
817 # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
818 samplingKey =
819
820 output =
821
822 # configure default thruput per second for this files group
823 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
824 # from the files in this group to 128Kb per second.
825 thruputPerSecond =
826
827 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
828 # older than 7 days
829 tooOldEvents =
830
831 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
832 tooNewEvents =
833
834 # by default every new event should start from not space symbol
835 eventPattern =
836
837 # Blacklisting and whitelisting the logs
838 # whitelist.0 = ^regexp$
839 # blacklist.0 = ^regexp$
840
841 # Lines matching a thruputExempt pattern pass every thruput cap and sampling; with one set,
842 # a spent cap drops the other lines instead of pausing this input
843 # thruputExempt.0 = ^regexp$
844
845 # Group each file's repeated lines into one event per run that counts them.
846 # Off unless true.
847 repeatEnabled = false
848
849 # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
850 # repeatMaxSpan = 1m
851
852 # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
853 # repeatOffsets = false
854
855 # What each file may remember: distinct lines, lines one event stands for, and bytes
856 # repeatMaxKeys = 256
857 # repeatMaxCount = 1000
858 # repeatMaxBytes = 256KB
859
860 # Fold each file's lines of one pattern past foldMaxPerMinute into one summary a minute.
861 # Lossy: folded lines are not sent. Off unless true.
862 foldEnabled = false
863
864 # Lines of one pattern a minute sent before the rest fold
865 # foldMaxPerMinute = 60
866
867 # Lines never folded (a regular expression; empty: the default)
868 # foldKeep = (?i)error|exception|fatal|panic
869
870 # What each file may keep a minute: its patterns and summaries
871 # foldMaxBytes = 256KB
872
873 # Report each file's top patterns every interval, as events of their own.
874 # Off unless true.
875 patternsEnabled = false
876
877 # How often a report goes out, from 1m to 24h
878 # patternsInterval = 15m
879
880 # How many patterns a report carries, up to 100
881 # patternsTop = 20
882
883 # What each file's patterns may keep; past it a new one takes the place of the lightest
884 # patternsMaxBytes = 256KB
885
886 # The report events' type (empty: the input's type with _patterns after it)
887 # patternsType =
888
889 [input.files::syslog]
890
891 disabled = true
892
893 path = /rootfs/var/log/
894
895 glob =
896
897 match = ^(syslog|messages|auth\.log)(.[\d\-]+)?$
898
899 # limit search only on one level
900 recursive = false
901
902 # files are read using polling schema, when reach the EOF how often to check if files got updated
903 pollingInterval = 250ms
904
905 # how often o look for the new files under logs path
906 walkingInterval = 5s
907
908 # include verbose fields in events (file offset)
909 verboseFields = false
910
911 type = k8s_host_logs
912
913 syslog.format = {{type}}|{{timestamp::format(2006-01-02T15:04:05.999999999Z07:00)}}|{{cluster}}|{{host}}|{{file_name}}|{{message}}
914
915 extraction = ^(?P<timestamp>[A-Za-z]+\s+\d+\s\d+:\d+:\d+|\d{4}-\d{2}-\d{2}T\S+)\s(?P<syslog_hostname>[^\s]+)\s(?P<syslog_component>[^:\[]+)(\[(?P<syslog_pid>\d+)\])?: (.+)$
916 extractionMessageField =
917
918 # the extracted fields naming the process that wrote a line, which groups its lines apart
919 extractionWriterFields = syslog_hostname, syslog_component, syslog_pid
920
921 timestampField = timestamp
922
923 # format for timestamp
924 # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006`
925 timestampFormat = @syslog
926
927 # Adjust date, if month/day aren't set in format
928 timestampSetMonth = false
929 timestampSetDay = false
930
931 # timestamp location (if not defined by format)
932 timestampLocation = Local
933
934 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
935 samplingPercent = -1
936
937 # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
938 samplingKey =
939
940 output =
941
942 # configure default thruput per second for this files group
943 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
944 # from the files in this group to 128Kb per second.
945 thruputPerSecond =
946
947 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
948 # older than 7 days
949 tooOldEvents =
950
951 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
952 tooNewEvents =
953
954 # by default every new event should start from not space symbol
955 eventPattern =
956
957 # Blacklisting and whitelisting the logs
958 # whitelist.0 = ^regexp$
959 # blacklist.0 = ^regexp$
960
961 # Lines matching a thruputExempt pattern pass every thruput cap and sampling; with one set,
962 # a spent cap drops the other lines instead of pausing this input
963 # thruputExempt.0 = ^regexp$
964
965 # Group each file's repeated lines into one event per run that counts them.
966 # Off unless true.
967 repeatEnabled = false
968
969 # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
970 # repeatMaxSpan = 1m
971
972 # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
973 # repeatOffsets = false
974
975 # What each file may remember: distinct lines, lines one event stands for, and bytes
976 # repeatMaxKeys = 256
977 # repeatMaxCount = 1000
978 # repeatMaxBytes = 256KB
979
980 # Fold each file's lines of one pattern past foldMaxPerMinute into one summary a minute.
981 # Lossy: folded lines are not sent. Off unless true.
982 foldEnabled = false
983
984 # Lines of one pattern a minute sent before the rest fold
985 # foldMaxPerMinute = 60
986
987 # Lines never folded (a regular expression; empty: the default)
988 # foldKeep = (?i)error|exception|fatal|panic
989
990 # What each file may keep a minute: its patterns and summaries
991 # foldMaxBytes = 256KB
992
993 # Report each file's top patterns every interval, as events of their own.
994 # Off unless true.
995 patternsEnabled = false
996
997 # How often a report goes out, from 1m to 24h
998 # patternsInterval = 15m
999
1000 # How many patterns a report carries, up to 100
1001 # patternsTop = 20
1002
1003 # What each file's patterns may keep; past it a new one takes the place of the lightest
1004 # patternsMaxBytes = 256KB
1005
1006 # The report events' type (empty: the input's type with _patterns after it)
1007 # patternsType =
1008
1009 [input.journald]
1010
1011 disabled = false
1012
1013 path.persistent = /rootfs/var/log/journal/
1014 path.volatile = /rootfs/run/log/journal/
1015
1016 # when reach end of journald, how often to pull
1017 pollingInterval = 250ms
1018
1019 # if you don't want to forward journald from the beginning,
1020 # set the oldest event in relative value, like -14h or -30m or -30s (h/m/s supported)
1021 startFromRel =
1022
1023 type = k8s_host_logs
1024
1025 syslog.format = {{type}}|{{timestamp::format(2006-01-02T15:04:05.999999999Z07:00)}}|{{cluster}}|{{host}}|journald|{{message}}
1026
1027 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
1028 samplingPercent = -1
1029
1030 # sampling key (should be regexp with the named match pattern `key`)
1031 samplingKey =
1032
1033 # how often to reopen the journald to free old files
1034 reopenInterval = 1h
1035
1036 output =
1037
1038 # configure default thruput per second for journald
1039 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
1040 # from the journald to 128Kb per second.
1041 thruputPerSecond =
1042
1043 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
1044 # older than 7 days
1045 tooOldEvents =
1046
1047 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
1048 tooNewEvents =
1049
1050 # by default every new event should start from not space symbol
1051 eventPattern = ^[^\s]
1052
1053 # By default ignoring verbose hyperkube logs (all INFO messages)
1054
1055 # Lines matching a thruputExempt pattern pass every thruput cap and sampling; with one set,
1056 # a spent cap drops the other lines instead of pausing journald
1057
1058 # Move Journald logs reader to a separate process, to prevent process from crashing in case of corrupted log files
1059 spawnExternalProcess = false
1060
1061 # Group each writer's repeated lines into one event per run that counts them.
1062 # Off unless true.
1063 # A writer is a unit, identifier, command, process, priority and facility: never grouped with another.
1064 repeatEnabled = false
1065
1066 # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
1067 # repeatMaxSpan = 1m
1068
1069 # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
1070 # repeatOffsets = false
1071
1072 # What the journal, all its writers together, may remember: distinct lines, lines one event stands for, and bytes
1073 # repeatMaxKeys = 256
1074 # repeatMaxCount = 1000
1075 # repeatMaxBytes = 256KB
1076
1077 # Fold each writer's lines of one pattern past foldMaxPerMinute into one summary a minute.
1078 # Lossy: folded lines are not sent. Off unless true.
1079 # A writer is a unit, identifier, command, process, priority and facility: never folded with another.
1080 foldEnabled = false
1081
1082 # Lines of one pattern a minute sent before the rest fold
1083 # foldMaxPerMinute = 60
1084
1085 # Lines never folded (a regular expression; empty: the default)
1086 # foldKeep = (?i)error|exception|fatal|panic
1087
1088 # What the journal, all its writers together, may keep a minute: its patterns and summaries
1089 # foldMaxBytes = 256KB
1090
1091 # Report the journal's top patterns every interval, as events of their own.
1092 # Off unless true.
1093 # A writer is a unit, identifier, command, process, priority and facility: its lines never share a pattern with another's, and the top patterns are of all its writers together.
1094 patternsEnabled = false
1095
1096 # How often a report goes out, from 1m to 24h
1097 # patternsInterval = 15m
1098
1099 # How many patterns a report carries, up to 100
1100 # patternsTop = 20
1101
1102 # What the journal's patterns may keep; past it a new one takes the place of the lightest
1103 # patternsMaxBytes = 256KB
1104
1105 # The report events' type (empty: the input's type with _patterns after it)
1106 # patternsType =
1107
1108 # Pipe to join events (container logs only)
1109 [pipe.join]
1110
1111 disabled = false
1112
1113 maxInterval = 100ms
1114
1115 maxWait = 1s
1116
1117 maxSize = 1MB
1118
1119 # Default pattern to indicate new message (should start not from space)
1120 patternRegex = ^[^\s]
1121
1122 # You can configure global replace rules for the events, which can help to remove sensitive data
1123 # from logs before they are sent to outputs. Those rules will be applied to all pipelines for container logs, host logs,
1124 # application logs and events.
1125 # In the following example we replace password=TEST with password=********
1126 ; [pipe.replace::name]
1127 ; patternRegex = (password=)([^\s]+)
1128 ; replace = $1********
1129
1130 # You can configure global hash rules for the events, which can help to hide sensitive data
1131 # from logs before they are sent to outputs. Those rules will be applied to all pipelines for container logs, host logs,
1132 # application logs and events.
1133 # In the following example we hash IP addresses with fnv-1a-64
1134 ; [pipe.hash::name]
1135 ; match = (\d{1,3}\.){3}\d{1,3}'
1136 ; function = fnv-1a-64
1137
1138 # Log lines matching patternRegex pass every thruput cap and sampling, in the workloads whose
1139 # fields match every matchRegex.<field>; over its cap such a workload drops its other lines
1140 # instead of pausing. Select by namespace: pod, container and image names are the workload
1141 # owner's to choose, and label fields never match
1142 ; [pipe.thruput_exempt::payments-audit]
1143 ; matchRegex.openshift_namespace = ^payments$
1144 ; patternRegex = ^AUDIT
1145
1146 # Count the events and bytes leaving every pipeline (pipeline_events, pipeline_bytes);
1147 # maxSeries bounds the label combinations. Fields are written with underscores.
1148 ; [pipe.eventcount]
1149 ; labels = index, sourcetype, output
1150 ; fields = openshift_namespace
1151 ; maxSeries = 10001152 004-addon.conf: |1153 [general]
1154 # addons can be run in parallel with agents
1155 addon = true
1156
1157 [input.kubernetes_events]
1158
1159 disabled = false
1160
1161 type = k8s_events
1162
1163 syslog.format = {{type}}|{{timestamp::format(2006-01-02T15:04:05.999999999Z07:00)}}|{{cluster}}|{{message}}
1164
1165 output =
1166
1167 excludeManagedFields = true1168---
1169apiVersion: apps/v1
1170kind: DaemonSet
1171metadata:
1172 name: collectorforopenshift-syslog
1173 namespace: collectorforopenshift-syslog
1174 annotations:
1175 collectord.collectord.io/logs-replace.ignoredebuglog-search: ^(TRACE|DEBUG) .*
1176 collectord.collectord.io/logs-replace.ignoredebuglog-val: ""
1177 labels:
1178 app: collectorforopenshift-syslog
1179spec:
1180 updateStrategy:
1181 type: RollingUpdate
1182 selector:
1183 matchLabels:
1184 daemon: collectorforopenshift-syslog
1185 template:
1186 metadata:
1187 name: collectorforopenshift-syslog
1188 annotations:
1189 # A change to the configuration files these pods mount rolls them:
1190 # Collectord reads its inputs' configuration at startup. A ConfigMap managed
1191 # outside the chart (configMap.create: false) gets no checksum: roll the pods
1192 # with an annotation of your own under podAnnotations.
1193 checksum/config: 9d00cded5719962da7d417a3f61d2916a192561d877ff5af24a1db9a4a91191b
1194 labels:
1195 daemon: collectorforopenshift-syslog
1196 spec:
1197 priorityClassName: "system-node-critical"
1198 dnsPolicy: ClusterFirstWithHostNet
1199 hostNetwork: true
1200 serviceAccountName: collectorforopenshift-syslog
1201 nodeSelector:
1202 kubernetes.io/os: linux
1203 affinity:
1204 nodeAffinity:
1205 requiredDuringSchedulingIgnoredDuringExecution:
1206 nodeSelectorTerms:
1207 - matchExpressions:
1208 - key: eks.amazonaws.com/compute-type
1209 operator: NotIn
1210 values:
1211 - fargate
1212 - key: type
1213 operator: NotIn
1214 values:
1215 - virtual-kubelet
1216 tolerations:
1217 - effect: NoSchedule
1218 operator: Exists
1219 - effect: NoExecute
1220 operator: Exists
1221 containers:
1222 - name: collectorforopenshift-syslog
1223 image: registry.connect.redhat.com/outcoldsolutions/collectorforopenshift:26.10.1
1224 imagePullPolicy: Always
1225 securityContext:
1226 runAsUser: 0
1227 privileged: true
1228 resources:
1229 limits:
1230 cpu: 2000m
1231 memory: 1024Mi
1232 requests:
1233 cpu: 500m
1234 memory: 256Mi
1235 env:
1236 - name: KUBERNETES_NODENAME
1237 valueFrom:
1238 fieldRef:
1239 fieldPath: spec.nodeName
1240 - name: KUBERNETES_NODEIP
1241 valueFrom:
1242 fieldRef:
1243 fieldPath: status.hostIP
1244 - name: POD_NAME
1245 valueFrom:
1246 fieldRef:
1247 fieldPath: metadata.name
1248 volumeMounts:
1249 - name: collectorforopenshift-syslog-state
1250 mountPath: /data
1251 - name: collectorforopenshift-syslog-config
1252 mountPath: /config/
1253 readOnly: true
1254 - name: rootfs
1255 mountPath: /rootfs/
1256 readOnly: false
1257 mountPropagation: HostToContainer
1258 - name: localtime
1259 mountPath: /etc/localtime
1260 readOnly: true
1261
1262 volumes:
1263 - name: collectorforopenshift-syslog-state
1264 hostPath:
1265 path: /var/lib/collectorforopenshift-syslog/data/
1266 type: DirectoryOrCreate
1267 - name: rootfs
1268 hostPath:
1269 path: /
1270 - name: localtime
1271 hostPath:
1272 path: /etc/localtime
1273 - name: collectorforopenshift-syslog-config
1274 configMap:
1275 name: collectorforopenshift-syslog
1276 items:
1277 - key: 001-general.conf
1278 path: 001-general.conf
1279 - key: 002-daemonset.conf
1280 path: 002-daemonset.conf
1281---
1282apiVersion: apps/v1
1283kind: Deployment
1284metadata:
1285 name: collectorforopenshift-syslog-addon
1286 namespace: collectorforopenshift-syslog
1287 annotations:
1288 collectord.collectord.io/logs-replace.ignoredebuglog-search: ^(TRACE|DEBUG) .*
1289 collectord.collectord.io/logs-replace.ignoredebuglog-val: ""
1290 labels:
1291 app: collectorforopenshift-syslog
1292spec:
1293 replicas: 1
1294 selector:
1295 matchLabels:
1296 daemon: collectorforopenshift-syslog
1297 template:
1298 metadata:
1299 name: collectorforopenshift-syslog-addon
1300 annotations:
1301 # A change to the configuration files these pods mount rolls them:
1302 # Collectord reads its inputs' configuration at startup. A ConfigMap managed
1303 # outside the chart (configMap.create: false) gets no checksum: roll the pods
1304 # with an annotation of your own under podAnnotations.
1305 checksum/config: d53cb2617ec22ccae3b92fe0131e7346927dde0af29ec593fad9dca72e643856
1306 labels:
1307 daemon: collectorforopenshift-syslog
1308 spec:
1309 priorityClassName: "system-cluster-critical"
1310 serviceAccountName: collectorforopenshift-syslog
1311 nodeSelector:
1312 kubernetes.io/os: linux
1313 affinity:
1314 nodeAffinity:
1315 requiredDuringSchedulingIgnoredDuringExecution:
1316 nodeSelectorTerms:
1317 - matchExpressions:
1318 - key: eks.amazonaws.com/compute-type
1319 operator: NotIn
1320 values:
1321 - fargate
1322 - key: type
1323 operator: NotIn
1324 values:
1325 - virtual-kubelet
1326 containers:
1327 - name: collectorforopenshift-syslog
1328 image: registry.connect.redhat.com/outcoldsolutions/collectorforopenshift:26.10.1
1329 imagePullPolicy: Always
1330 securityContext:
1331 allowPrivilegeEscalation: false
1332 capabilities:
1333 drop:
1334 - ALL
1335 privileged: false
1336 runAsUser: 0
1337 seLinuxOptions:
1338 type: spc_t
1339 seccompProfile:
1340 type: RuntimeDefault
1341 resources:
1342 limits:
1343 cpu: 1000m
1344 memory: 512Mi
1345 requests:
1346 cpu: 200m
1347 memory: 64Mi
1348 env:
1349 - name: KUBERNETES_NODENAME
1350 valueFrom:
1351 fieldRef:
1352 fieldPath: spec.nodeName
1353 - name: POD_NAME
1354 valueFrom:
1355 fieldRef:
1356 fieldPath: metadata.name
1357 volumeMounts:
1358 - name: collectorforopenshift-syslog-state
1359 mountPath: /data
1360 - name: collectorforopenshift-syslog-config
1361 mountPath: /config/
1362 readOnly: true
1363 - name: localtime
1364 mountPath: /etc/localtime
1365 readOnly: true
1366
1367 volumes:
1368 # The DaemonSet pod on this node owns the directory and the database the
1369 # addon opens read-only. DirectoryOrCreate, so an addon scheduled first
1370 # mounts it too; Collectord waits for the DaemonSet to create the database.
1371 - name: collectorforopenshift-syslog-state
1372 hostPath:
1373 path: /var/lib/collectorforopenshift-syslog/data/
1374 type: DirectoryOrCreate
1375 - name: localtime
1376 hostPath:
1377 path: /etc/localtime
1378 - name: collectorforopenshift-syslog-config
1379 configMap:
1380 name: collectorforopenshift-syslog
1381 items:
1382 - key: 001-general.conf
1383 path: 001-general.conf
1384 - key: 004-addon.conf
1385 path: 004-addon.conf