Collectord configuration for Kubernetes

  1apiVersion: v1
  2kind: Namespace
  3metadata:
  4  labels:
  5    app: collectorforkubernetes-syslog
  6  name: collectorforkubernetes-syslog
  7---
  8apiVersion: apiextensions.k8s.io/v1
  9kind: CustomResourceDefinition
 10metadata:
 11  name: configurations.collectord.io
 12spec:
 13  group: collectord.io
 14  versions:
 15    - name: v1
 16      served: true
 17      storage: true
 18      schema:
 19        openAPIV3Schema:
 20          type: object
 21          properties:
 22            spec:
 23              type: object
 24              additionalProperties: true
 25            force:
 26              type: boolean
 27  scope: Cluster
 28  names:
 29    listKind: ConfigurationList
 30    plural: configurations
 31    singular: configuration
 32    kind: Configuration
 33---
 34apiVersion: apiextensions.k8s.io/v1
 35kind: CustomResourceDefinition
 36metadata:
 37  name: splunkoutputs.collectord.io
 38spec:
 39  group: collectord.io
 40  versions:
 41    - name: v1
 42      served: true
 43      storage: true
 44      schema:
 45        openAPIV3Schema:
 46          type: object
 47          properties:
 48            spec:
 49              type: object
 50              properties:
 51                url:
 52                  type: string
 53                  format: uri
 54                insecure:
 55                  type: boolean
 56                token:
 57                  type: string
 58                  description: "Plain token"
 59                tokenFromSecret:
 60                  type: object
 61                  description: "Reference to a Kubernetes Secret"
 62                  properties:
 63                    secret:
 64                      type: string
 65                    key:
 66                      type: string
 67                defaultMetricsIndex:
 68                  type: string
 69                  description: "Index of the metrics-index records sent through this output; empty for the token's default"
 70                incorrectIndexBehavior:
 71                  type: string
 72                  description: "What happens to a record the token refuses"
 73                  enum: ["RedirectToDefault", "Drop", "Retry"]
 74                requireExplicitIndex:
 75                  type: boolean
 76                  description: "Drop a record that carries no index instead of sending it to the token's default"
 77              oneOf:
 78                - required: ["token"]
 79                - required: ["tokenFromSecret"]
 80  scope: Namespaced
 81  names:
 82    listKind: SplunkOutputList
 83    plural: splunkoutputs
 84    singular: splunkoutput
 85    kind: SplunkOutput
 86---
 87apiVersion: v1
 88kind: ServiceAccount
 89metadata:
 90  labels:
 91    app: collectorforkubernetes-syslog
 92  name: collectorforkubernetes-syslog
 93  namespace: collectorforkubernetes-syslog
 94---
 95apiVersion: rbac.authorization.k8s.io/v1
 96kind: ClusterRole
 97metadata:
 98  name: collectorforkubernetes-syslog
 99  labels:
100    app: collectorforkubernetes-syslog
101rules:
102  - apiGroups:
103    - ""
104    resources:
105    - events
106    - namespaces
107    - nodes
108    - nodes/metrics
109    - persistentvolumeclaims
110    - persistentvolumes
111    - pods
112    - replicationcontrollers
113    - resourcequotas
114    - services
115    verbs:
116    - get
117    - list
118    - watch
119  - apiGroups:
120    - apps
121    resources:
122    - daemonsets
123    - deployments
124    - replicasets
125    - statefulsets
126    verbs:
127    - get
128    - list
129    - watch
130  - apiGroups:
131    - batch
132    resources:
133    - cronjobs
134    - jobs
135    verbs:
136    - get
137    - list
138    - watch
139  - apiGroups:
140    - discovery.k8s.io
141    resources:
142    - endpointslices
143    verbs:
144    - get
145    - list
146    - watch
147  - apiGroups:
148    - rbac.authorization.k8s.io
149    resources:
150    - clusterroles
151    verbs:
152    - get
153    - list
154    - watch
155  - apiGroups:
156    - collectord.io
157    resources:
158    - configurations
159    - splunkoutputs
160    verbs:
161    - get
162    - list
163    - watch
164  - apiGroups: []
165    nonResourceURLs:
166    - /metrics
167    resources: []
168    verbs:
169    - get
170  - apiGroups: []
171    nonResourceURLs:
172    - /api
173    - /api/*
174    - /apis
175    - /apis/*
176    resources: []
177    verbs:
178    - get
179---
180apiVersion: rbac.authorization.k8s.io/v1
181kind: ClusterRoleBinding
182metadata:
183  name: collectorforkubernetes-syslog
184  labels:
185    app: collectorforkubernetes-syslog
186roleRef:
187  apiGroup: rbac.authorization.k8s.io
188  kind: ClusterRole
189  name: collectorforkubernetes-syslog
190subjects:
191  - kind: ServiceAccount
192    name: collectorforkubernetes-syslog
193    namespace: collectorforkubernetes-syslog
194---
195apiVersion: v1
196kind: ResourceQuota
197metadata:
198  name: collectorforkubernetes-syslog-critical-pods
199  namespace: collectorforkubernetes-syslog
200  labels:
201    app: collectorforkubernetes-syslog
202spec:
203  hard:
204    pods: 1G
205  scopeSelector:
206    matchExpressions:
207    - scopeName: PriorityClass
208      operator: In
209      values:
210      - system-cluster-critical
211      - system-node-critical
212---
213apiVersion: v1
214kind: ConfigMap
215metadata:
216  name: collectorforkubernetes-syslog
217  namespace: collectorforkubernetes-syslog
218  labels:
219    app: collectorforkubernetes-syslog
220data:
221  001-general.conf: |

222    # The general configuration is used for all deployments
223    #
224    # Run collectord with the flag -conf and specify location of the configuration files.
225    #
226    # You can override all the values using environment variables with the format like
227    #   COLLECTOR__<ANYNAME>=<section>__<key>=<value>
228    # As an example you can set dataPath in [general] section as
229    #   COLLECTOR__DATAPATH=general__dataPath=C:\\some\\path\\data.db
230    # This parameter can be configured using -env-override, set it to empty string to disable this feature
231
232    [general]
233
234    # Please review license https://www.outcoldsolutions.com/legal/license-agreement/
235    # and accept license by changing the value to *true*
236    acceptLicense = false
237
238    # Location for the database
239    # Collectord stores positions of the files and internal state
240    dataPath = ./data/
241
242    # log level (accepted values are trace, debug, info, warn, error, fatal)
243    logLevel = info
244
245    # Internal http server - serves the liveness (/healthz) and readiness (/readyz)
246    # probes. DaemonSets run on the host network, so bind to loopback (127.0.0.1:11888)
247    # to keep the endpoints off the node IP; the kubelet still reaches them via
248    # httpGet.host 127.0.0.1. The addon runs off the host network and overrides the
249    # bind to 0.0.0.0 in 004-addon.conf so the kubelet can reach it on the pod IP.
250    # httpServerBinding = 0.0.0.0:11888
251    httpServerBinding =
252
253    httpServerLog = false
254
255    # telemetry report endpoint, set it to empty string to disable telemetry
256    telemetryEndpoint = https://license.outcold.solutions/telemetry/
257
258    licenseEndpoint = https://license.outcold.solutions/license/
259
260    # Only for the Outcold Solutions License Server; one on-premises uses [license.client]
261    licenseServerProxyUrl =
262
263    # user:password
264    licenseServerProxyBasicAuth =
265
266    license =
267
268    # Environment variable $KUBERNETES_NODENAME is used by default to setup hostname
269    # Use value below to override specific name
270    hostname =
271
272    # Default output for events, logs and metrics
273    # valid values: splunk and devnull
274    # Use devnull by default if you don't want to redirect data
275    defaultOutput = syslog
276
277    fileInputBufferSize = 8kb
278
279    fileInputLineMaxSize = 1mb
280
281    # Include custom fields to attach to every event, in example below every event sent to Splunk will hav
282    # indexed field my_environment=dev. Fields names should match to ^[a-z][_a-z0-9]*$
283    # Better way to configure that is to specify labels for Kubernetes Nodes.
284    # ; fields.my_environment = dev
285    # Identify the cluster if you are planning to monitor multiple clusters
286    fields.cluster = -
287
288    # Include EC2 Metadata (see list of possible fields https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
289    # Should be in format ec2Metadata.{desired_field_name} = {url path to read the value}
290    # ec2Metadata.ec2_instance_id = /latest/meta-data/instance-id
291    # ec2Metadata.ec2_instance_type = /latest/meta-data/instance-type
292
293    # subdomain for the annotations added to the pods, workloads, namespaces or containers, like splunk.collectord.io/..
294    annotationsSubdomain = syslog
295
296    # configure global thruput per second for forwarded logs (metrics are not included)
297    # for example if you set `thruputPerSecond = 512Kb`, that will limit amount of logs forwarded
298    # from the single Collectord instance to 512Kb per second.
299    # You can configure thruput individually for the logs (including specific for container logs) below
300    thruputPerSecond =
301
302    # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
303    # older than 7 days
304    tooOldEvents =
305
306    # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
307    tooNewEvents =
308
309    # How Collectord spells the fields it composes: "_" (kubernetes_pod_name, the default) or "."
310    fieldsSeparator =
311
312    # For input.files::X and application logs, when glob or match are configured, Collectord can automatically
313    # detect gzipped files and skip them (based on the extensions or magic numbers)
314    autoSkipGzipFiles = true
315
316    # Publish to non-default outputs asynchronously (default), so a slow or down output does not
317    # block the others; false holds the pipeline back instead of dropping events.
318    multioutput.async = true
319
320    # Buffer size for the async proxy (default 100), per output of each pipeline. When it and
321    # the output's queue are full, events are dropped (multioutput_dropped_events).
322    multioutput.asyncBufferSize = 100
323
324    [license.client]
325    # point to the license located on the HTTP web server, or a hosted by the Collectord running as license server
326    url =
327    basicAuth =
328    insecure = false
329    capath =
330    caname =
331    proxyUrl =
332
333    # user:password
334    proxyBasicAuth =
335
336    [general.kubernetes]
337
338    # Override service URL for Kubernetes (default is ${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT})
339    serviceURL =
340
341    # Environment variable $KUBERNETES_NODENAME is used by default to setup nodeName
342    # Use it only when you need to override it
343    nodeName =
344
345    tokenPath = /var/run/secrets/kubernetes.io/serviceaccount/token
346    certPath = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
347
348    # Default timeout for http responses. The streaming/watch requests depend on this timeout.
349    timeout = 30m
350
351    # How long to keep the cache for the recent calls to API server (to limit number of calls when collectord discovers new pods)
352    metadataTTL = 30s
353
354    podsCgroupFilter = ^/([^/\s]+/)*kubepods(\.slice)?/((kubepods-)?(burstable|besteffort)(\.slice)?/)?([^/]*)pod([0-9a-f]{32}|[0-9a-f\-_]{36})(\.slice)?$
355
356    containersCgroupFilter = ^/([^/\s]+/)*kubepods(\.slice)?/((kubepods-)?(burstable|besteffort)(\.slice)?/)?([^/]*)pod([0-9a-f]{32}|[0-9a-f\-_]{36})(\.slice)?/(docker-|crio-|cri-\w+-)?[0-9a-f]{64}(\.scope)?(\/.+)?$
357
358    # path to the kubelet root location (use it to discover application logs for emptyDir)
359    # the expected format is `pods/{pod-id}/volumes/kubernetes.io~empty-dir/{volume-name}/_data/`
360    volumesRootDir = /rootfs/var/lib/kubelet/
361
362    # You can attach annotations as a metadata, using the format
363    #   includeAnnotations.{key} = {regexp}
364    # For example if you want to include all annotations that starts with `prometheus.io` or `example.com` you can include
365    # the following format:
366    #   includeAnnotations.1 = ^prometheus\.io.*
367    #   includeAnnotations.2 = ^example\.com.*
368
369    # You can exclude labels from metadata, using the format
370    #   excludeLabels.{key} = {regexp}
371    # For example if you want to exclude all labels that starts with `prometheus.io` or `example.com` you can include
372    # the following format:
373    #   excludeLabels.1 = ^prometheus\.io.*
374    #   excludeLabels.2 = ^example\.com.*
375
376    # watch for changes (annotations) in the objects
377    watch.namespace = v1/namespace
378    watch.deployment = apps/v1/deployment
379    watch.configuration = collectord.io/v1/configuration
380
381    # Collectord can review the assigned ClusterRole and traverse metadata for the Pods only for the Owner objects
382    # that are defined in the ClusterRole, ignoring anything else, it does not have access to.
383    # This way Collectord does not generate 403 requests on API Server
384    clusterRole = collectorforkubernetes-syslog
385
386    # Alternative of telling Collectord about the ClusterRole is to manually list the objects.
387    # You can define which objects Collectord should traverse when it sees Owners.
388    ; traverseOwnership.namespaces = v1/namespace
389
390    [output.syslog]
391
392    # tcp or udp
393    network = tcp
394    address =

395  002-daemonset.conf: |

 396    [general.cri-o]
 397
 398    # url for CRIO API, only unix socket is supported
 399    url = unix:///rootfs/var/run/crio/crio.sock
 400
 401    # Timeout for http responses to docker client. The streaming requests depend on this timeout.
 402    timeout = 1m
 403
 404    [general.containerd]
 405    # Runtime can be on /rootfs/run/containerd (depends on the Linux distribution)
 406    runtimePath = /rootfs/var/run/containerd
 407    namespace = k8s.io
 408
 409    [input.files]
 410
 411    disabled = false
 412
 413    # root location of docker log files
 414    # logs are expected in standard docker format like {containerID}/{containerID}-json.log
 415    # rotated files
 416    path = /rootfs/var/lib/docker/containers/
 417
 418    # root location of CRI-O files
 419    # logs are expected in Kubernetes format, like {podID}/{containerName}/0.log
 420    crioPath = /rootfs/var/log/pods/
 421
 422    # files are read using polling schema, when reach the EOF how often to check if files got updated
 423    pollingInterval = 250ms
 424
 425    # how often to look for the new files under logs path
 426    walkingInterval = 5s
 427
 428    # include verbose fields in events (file offset)
 429    verboseFields = false
 430
 431    type = k8s_logs
 432
 433    syslog.format = {{type}}|{{timestamp::format(2006-01-02T15:04:05.999999999Z07:00)}}|{{cluster}}|{{host}}|{{namespace}}|{{pod_id}}|{{pod_name}}|{{container_name}}|{{stream}}|{{message}}
 434
 435    # docker splits events when they are larger than 10-100k (depends on the docker version)
 436    # we join them together by default and forward to Splunk as one event
 437    joinPartialEvents = true
 438
 439    # In case if your containers report messages with terminal colors or other escape sequences
 440    # you can enable strip for all the containers in one place.
 441    # Better is to enable it only for required container with the annotation syslog.collectord.io/logs-escapeterminalsequences=true
 442    stripTerminalEscapeSequences = false
 443
 444    # Regexp used for stripping terminal colors, it does not stip all the escape sequences
 445    # Read http://man7.org/linux/man-pages/man4/console_codes.4.html for more information
 446    stripTerminalEscapeSequencesRegex = (\x1b\[\d{1,3}(;\d{1,3})*m)|(\x07)|(\x1b]\d+(\s\d)?;[^\x07]+\x07)|(.*\x1b\[K)
 447
 448    # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
 449    samplingPercent = -1
 450
 451    # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
 452    samplingKey =
 453
 454    output =
 455
 456    # configure default thruput per second for for each container log
 457    # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
 458    # from the single container to 128Kb per second.
 459    thruputPerSecond =
 460
 461    # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
 462    # older than 7 days
 463    tooOldEvents =
 464
 465    # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
 466    tooNewEvents =
 467
 468    # Read rotated logs the kubelet compressed before Collectord read them; false counts them lost
 469    readCompressed = true
 470
 471    # Bytes of those logs inflated at once, node-wide, under [general] dataPath
 472    compressedInflateBudget = 256MiB
 473
 474    # Group each stream's repeated lines into one event per run that counts them.
 475    # Off unless true; a workload overrides it with its
 476    # syslog.collectord.io/logs-repeat-enabled annotation.
 477    repeatEnabled = false
 478
 479    # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
 480    # repeatMaxSpan = 1m
 481
 482    # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
 483    # repeatOffsets = false
 484
 485    # What each stream may remember: distinct lines, lines one event stands for, and bytes
 486    # repeatMaxKeys = 256
 487    # repeatMaxCount = 1000
 488    # repeatMaxBytes = 256KB
 489
 490    # Fold each stream's lines of one pattern past foldMaxPerMinute into one summary a minute.
 491    # Lossy: folded lines are not sent. Off unless true; a workload overrides it with its
 492    # syslog.collectord.io/logs-fold-enabled annotation.
 493    foldEnabled = false
 494
 495    # Lines of one pattern a minute sent before the rest fold
 496    # foldMaxPerMinute = 60
 497
 498    # Lines never folded (a regular expression; empty: the default)
 499    # foldKeep = (?i)error|exception|fatal|panic
 500
 501    # What each stream may keep a minute: its patterns and summaries
 502    # foldMaxBytes = 256KB
 503
 504    # Report each stream's top patterns every interval, as events of their own.
 505    # Off unless true; a workload overrides it with its
 506    # syslog.collectord.io/logs-patterns-enabled annotation.
 507    patternsEnabled = false
 508
 509    # How often a report goes out, from 1m to 24h
 510    # patternsInterval = 15m
 511
 512    # How many patterns a report carries, up to 100
 513    # patternsTop = 20
 514
 515    # What each stream's patterns may keep; past it a new one takes the place of the lightest
 516    # patternsMaxBytes = 256KB
 517
 518    # The report events' type (empty: the input's type with _patterns after it)
 519    # patternsType =
 520
 521    [input.app_logs]
 522
 523    disabled = false
 524
 525    # root location of mounts (applies to hostPath mounts only), if the hostPath differs inside container from the path on host
 526    root = /rootfs/
 527
 528    # how often to review list of available volumes
 529    syncInterval = 5s
 530
 531    glob = *.log*
 532
 533    # files are read using polling schema, when reach the EOF how often to check if files got updated
 534    pollingInterval = 250ms
 535
 536    # how often to look for the new files under logs path
 537    walkingInterval = 5s
 538
 539    # include verbose fields in events (file offset)
 540    verboseFields = false
 541
 542    type = k8s_logs
 543
 544    syslog.format = {{type}}|{{timestamp::format(2006-01-02T15:04:05.999999999Z07:00)}}|{{cluster}}|{{host}}|{{namespace}}|{{pod_id}}|{{pod_name}}|{{container_name}}|{{file_name}}|{{message}}
 545
 546    # we split files using new line character, with this configuration you can specify what defines the new event
 547    # after new line
 548    eventPatternRegex = ^[^\s]
 549    eventPatternMaxInterval = 100ms
 550    eventPatternMaxWait = 1s
 551    eventPatternMaxSize = 1MB
 552
 553    # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
 554    samplingPercent = -1
 555
 556    # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
 557    samplingKey =
 558
 559    output =
 560
 561    # configure default thruput per second for for each container log
 562    # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
 563    # from the single container to 128Kb per second.
 564    thruputPerSecond =
 565
 566    # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
 567    # older than 7 days
 568    tooOldEvents =
 569
 570    # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
 571    tooNewEvents =
 572
 573    # Configure how long Collectord should keep the file descriptors open for files, that has not been forwarded yet
 574    # When using PVC, and if pipeline is lagging behind, Collectord holding open fd for files, can cause long termination
 575    # of pods, as kubelet cannot unmount the PVC volume from the system
 576    maxHoldAfterClose = 1800s
 577
 578    # Group each file's repeated lines into one event per run that counts them.
 579    # Off unless true; a workload overrides it with its
 580    # syslog.collectord.io/volume.<n>-logs-repeat-enabled annotation.
 581    repeatEnabled = false
 582
 583    # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
 584    # repeatMaxSpan = 1m
 585
 586    # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
 587    # repeatOffsets = false
 588
 589    # What each file may remember: distinct lines, lines one event stands for, and bytes
 590    # repeatMaxKeys = 256
 591    # repeatMaxCount = 1000
 592    # repeatMaxBytes = 256KB
 593
 594    # Fold each file's lines of one pattern past foldMaxPerMinute into one summary a minute.
 595    # Lossy: folded lines are not sent. Off unless true; a workload overrides it with its
 596    # syslog.collectord.io/volume.<n>-logs-fold-enabled annotation.
 597    foldEnabled = false
 598
 599    # Lines of one pattern a minute sent before the rest fold
 600    # foldMaxPerMinute = 60
 601
 602    # Lines never folded (a regular expression; empty: the default)
 603    # foldKeep = (?i)error|exception|fatal|panic
 604
 605    # What each file may keep a minute: its patterns and summaries
 606    # foldMaxBytes = 256KB
 607
 608    # Report each file's top patterns every interval, as events of their own.
 609    # Off unless true; a workload overrides it with its
 610    # syslog.collectord.io/volume.<n>-logs-patterns-enabled annotation.
 611    patternsEnabled = false
 612
 613    # How often a report goes out, from 1m to 24h
 614    # patternsInterval = 15m
 615
 616    # How many patterns a report carries, up to 100
 617    # patternsTop = 20
 618
 619    # What each file's patterns may keep; past it a new one takes the place of the lightest
 620    # patternsMaxBytes = 256KB
 621
 622    # The report events' type (empty: the input's type with _patterns after it)
 623    # patternsType =
 624
 625    [input.files::audit-logs]
 626
 627    disabled = false
 628
 629    path = /rootfs/var/log/kube-apiserver/
 630
 631    glob = audit*.log
 632
 633    match =
 634
 635    # limit search only on one level
 636    recursive = false
 637    nodeSelector = node-role.kubernetes.io/control-plane
 638
 639    # files are read using polling schema, when reach the EOF how often to check if files got updated
 640    pollingInterval = 250ms
 641
 642    # how often o look for the new files under logs path
 643    walkingInterval = 5s
 644
 645    # include verbose fields in events (file offset)
 646    verboseFields = false
 647
 648    type = k8s_audit_logs
 649
 650    syslog.format = {{type}}|{{timestamp::format(2006-01-02T15:04:05.999999999Z07:00)}}|{{cluster}}|{{message}}
 651
 652    extraction = (?P<message>.*"stageTimestamp":"(?P<timestamp>[^"]+)".*)
 653    extractionMessageField = message
 654
 655    timestampField = timestamp
 656
 657    # format for timestamp
 658    # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006`
 659    timestampFormat = 2006-01-02T15:04:05.999999999Z07:00
 660
 661    # timestamp location (if not defined by format)
 662    timestampLocation =
 663
 664    # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
 665    samplingPercent = -1
 666
 667    # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
 668    samplingKey =
 669
 670    output =
 671
 672    # configure default thruput per second for this files group
 673    # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
 674    # from the files in this group to 128Kb per second.
 675    thruputPerSecond =
 676
 677    # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
 678    # older than 7 days
 679    tooOldEvents =
 680
 681    # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
 682    tooNewEvents =
 683
 684    # by default every new event should start from not space symbol
 685    eventPattern =
 686
 687    # Blacklisting and whitelisting the logs
 688    # whitelist.0 = ^regexp$
 689    # blacklist.0 = ^regexp$
 690
 691    # Lines matching a thruputExempt pattern pass every thruput cap and sampling; with one set,
 692    # a spent cap drops the other lines instead of pausing this input
 693    # thruputExempt.0 = ^regexp$
 694
 695    # Group each file's repeated lines into one event per run that counts them.
 696    # Off unless true.
 697    repeatEnabled = false
 698
 699    # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
 700    # repeatMaxSpan = 1m
 701
 702    # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
 703    # repeatOffsets = false
 704
 705    # What each file may remember: distinct lines, lines one event stands for, and bytes
 706    # repeatMaxKeys = 256
 707    # repeatMaxCount = 1000
 708    # repeatMaxBytes = 256KB
 709
 710    # Fold each file's lines of one pattern past foldMaxPerMinute into one summary a minute.
 711    # Lossy: folded lines are not sent. Off unless true.
 712    foldEnabled = false
 713
 714    # Lines of one pattern a minute sent before the rest fold
 715    # foldMaxPerMinute = 60
 716
 717    # Lines never folded (a regular expression; empty: the default)
 718    # foldKeep = (?i)error|exception|fatal|panic
 719
 720    # What each file may keep a minute: its patterns and summaries
 721    # foldMaxBytes = 256KB
 722
 723    # Report each file's top patterns every interval, as events of their own.
 724    # Off unless true.
 725    patternsEnabled = false
 726
 727    # How often a report goes out, from 1m to 24h
 728    # patternsInterval = 15m
 729
 730    # How many patterns a report carries, up to 100
 731    # patternsTop = 20
 732
 733    # What each file's patterns may keep; past it a new one takes the place of the lightest
 734    # patternsMaxBytes = 256KB
 735
 736    # The report events' type (empty: the input's type with _patterns after it)
 737    # patternsType =
 738
 739    [input.files::logs]
 740
 741    disabled = false
 742
 743    path = /rootfs/var/log/
 744
 745    glob =
 746
 747    match = ^(([\w\-.]+\.log(.[\d\-]+)?)|(docker))$
 748    exclude = ^(auth|cron|daemon|kern|lpr|mail|user|ufw)\.log(.[\d\-]+)?$
 749
 750    # files are read using polling schema, when reach the EOF how often to check if files got updated
 751    pollingInterval = 250ms
 752
 753    # how often o look for the new files under logs path
 754    walkingInterval = 5s
 755
 756    # include verbose fields in events (file offset)
 757    verboseFields = false
 758
 759    type = k8s_host_logs
 760
 761    syslog.format = {{type}}|{{timestamp::format(2006-01-02T15:04:05.999999999Z07:00)}}|{{cluster}}|{{host}}|{{file_name}}|{{message}}
 762
 763    extraction =
 764    extractionMessageField =
 765
 766    timestampField =
 767
 768    # format for timestamp
 769    # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006`
 770    timestampFormat =
 771
 772    # timestamp location (if not defined by format)
 773    timestampLocation =
 774
 775    # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
 776    samplingPercent = -1
 777
 778    # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
 779    samplingKey =
 780
 781    output =
 782
 783    # configure default thruput per second for this files group
 784    # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
 785    # from the files in this group to 128Kb per second.
 786    thruputPerSecond =
 787
 788    # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
 789    # older than 7 days
 790    tooOldEvents =
 791
 792    # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
 793    tooNewEvents =
 794
 795    # by default every new event should start from not space symbol
 796    eventPattern =
 797
 798    # Blacklisting and whitelisting the logs
 799    # whitelist.0 = ^regexp$
 800    # blacklist.0 = ^regexp$
 801
 802    # Lines matching a thruputExempt pattern pass every thruput cap and sampling; with one set,
 803    # a spent cap drops the other lines instead of pausing this input
 804    # thruputExempt.0 = ^regexp$
 805
 806    # Group each file's repeated lines into one event per run that counts them.
 807    # Off unless true.
 808    repeatEnabled = false
 809
 810    # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
 811    # repeatMaxSpan = 1m
 812
 813    # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
 814    # repeatOffsets = false
 815
 816    # What each file may remember: distinct lines, lines one event stands for, and bytes
 817    # repeatMaxKeys = 256
 818    # repeatMaxCount = 1000
 819    # repeatMaxBytes = 256KB
 820
 821    # Fold each file's lines of one pattern past foldMaxPerMinute into one summary a minute.
 822    # Lossy: folded lines are not sent. Off unless true.
 823    foldEnabled = false
 824
 825    # Lines of one pattern a minute sent before the rest fold
 826    # foldMaxPerMinute = 60
 827
 828    # Lines never folded (a regular expression; empty: the default)
 829    # foldKeep = (?i)error|exception|fatal|panic
 830
 831    # What each file may keep a minute: its patterns and summaries
 832    # foldMaxBytes = 256KB
 833
 834    # Report each file's top patterns every interval, as events of their own.
 835    # Off unless true.
 836    patternsEnabled = false
 837
 838    # How often a report goes out, from 1m to 24h
 839    # patternsInterval = 15m
 840
 841    # How many patterns a report carries, up to 100
 842    # patternsTop = 20
 843
 844    # What each file's patterns may keep; past it a new one takes the place of the lightest
 845    # patternsMaxBytes = 256KB
 846
 847    # The report events' type (empty: the input's type with _patterns after it)
 848    # patternsType =
 849
 850    [input.files::syslog]
 851
 852    disabled = true
 853
 854    path = /rootfs/var/log/
 855
 856    glob =
 857
 858    match = ^(syslog|messages|auth\.log)(.[\d\-]+)?$
 859
 860    # limit search only on one level
 861    recursive = false
 862
 863    # files are read using polling schema, when reach the EOF how often to check if files got updated
 864    pollingInterval = 250ms
 865
 866    # how often o look for the new files under logs path
 867    walkingInterval = 5s
 868
 869    # include verbose fields in events (file offset)
 870    verboseFields = false
 871
 872    type = k8s_host_logs
 873
 874    syslog.format = {{type}}|{{timestamp::format(2006-01-02T15:04:05.999999999Z07:00)}}|{{cluster}}|{{host}}|{{file_name}}|{{message}}
 875
 876    extraction = ^(?P<timestamp>[A-Za-z]+\s+\d+\s\d+:\d+:\d+|\d{4}-\d{2}-\d{2}T\S+)\s(?P<syslog_hostname>[^\s]+)\s(?P<syslog_component>[^:\[]+)(\[(?P<syslog_pid>\d+)\])?: (.+)$
 877    extractionMessageField =
 878
 879    # the extracted fields naming the process that wrote a line, which groups its lines apart
 880    extractionWriterFields = syslog_hostname, syslog_component, syslog_pid
 881
 882    timestampField = timestamp
 883
 884    # format for timestamp
 885    # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006`
 886    timestampFormat = @syslog
 887
 888    # Adjust date, if month/day aren't set in format
 889    timestampSetMonth = false
 890    timestampSetDay = false
 891
 892    # timestamp location (if not defined by format)
 893    timestampLocation = Local
 894
 895    # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
 896    samplingPercent = -1
 897
 898    # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
 899    samplingKey =
 900
 901    output =
 902
 903    # configure default thruput per second for this files group
 904    # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
 905    # from the files in this group to 128Kb per second.
 906    thruputPerSecond =
 907
 908    # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
 909    # older than 7 days
 910    tooOldEvents =
 911
 912    # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
 913    tooNewEvents =
 914
 915    # by default every new event should start from not space symbol
 916    eventPattern =
 917
 918    # Blacklisting and whitelisting the logs
 919    # whitelist.0 = ^regexp$
 920    # blacklist.0 = ^regexp$
 921
 922    # Lines matching a thruputExempt pattern pass every thruput cap and sampling; with one set,
 923    # a spent cap drops the other lines instead of pausing this input
 924    # thruputExempt.0 = ^regexp$
 925
 926    # Group each file's repeated lines into one event per run that counts them.
 927    # Off unless true.
 928    repeatEnabled = false
 929
 930    # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
 931    # repeatMaxSpan = 1m
 932
 933    # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
 934    # repeatOffsets = false
 935
 936    # What each file may remember: distinct lines, lines one event stands for, and bytes
 937    # repeatMaxKeys = 256
 938    # repeatMaxCount = 1000
 939    # repeatMaxBytes = 256KB
 940
 941    # Fold each file's lines of one pattern past foldMaxPerMinute into one summary a minute.
 942    # Lossy: folded lines are not sent. Off unless true.
 943    foldEnabled = false
 944
 945    # Lines of one pattern a minute sent before the rest fold
 946    # foldMaxPerMinute = 60
 947
 948    # Lines never folded (a regular expression; empty: the default)
 949    # foldKeep = (?i)error|exception|fatal|panic
 950
 951    # What each file may keep a minute: its patterns and summaries
 952    # foldMaxBytes = 256KB
 953
 954    # Report each file's top patterns every interval, as events of their own.
 955    # Off unless true.
 956    patternsEnabled = false
 957
 958    # How often a report goes out, from 1m to 24h
 959    # patternsInterval = 15m
 960
 961    # How many patterns a report carries, up to 100
 962    # patternsTop = 20
 963
 964    # What each file's patterns may keep; past it a new one takes the place of the lightest
 965    # patternsMaxBytes = 256KB
 966
 967    # The report events' type (empty: the input's type with _patterns after it)
 968    # patternsType =
 969
 970    [input.journald]
 971
 972    disabled = false
 973
 974    path.persistent = /rootfs/var/log/journal/
 975    path.volatile = /rootfs/run/log/journal/
 976
 977    # when reach end of journald, how often to pull
 978    pollingInterval = 250ms
 979
 980    # if you don't want to forward journald from the beginning,
 981    # set the oldest event in relative value, like -14h or -30m or -30s (h/m/s supported)
 982    startFromRel =
 983
 984    type = k8s_host_logs
 985
 986    syslog.format = {{type}}|{{timestamp::format(2006-01-02T15:04:05.999999999Z07:00)}}|{{cluster}}|{{host}}|journald|{{message}}
 987
 988    # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
 989    samplingPercent = -1
 990
 991    # sampling key (should be regexp with the named match pattern `key`)
 992    samplingKey =
 993
 994    # how often to reopen the journald to free old files
 995    reopenInterval = 1h
 996
 997    output =
 998
 999    # configure default thruput per second for journald
1000    # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
1001    # from the journald to 128Kb per second.
1002    thruputPerSecond =
1003
1004    # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
1005    # older than 7 days
1006    tooOldEvents =
1007
1008    # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
1009    tooNewEvents =
1010
1011    # by default every new event should start from not space symbol
1012    eventPattern = ^[^\s]
1013
1014    # By default ignoring verbose hyperkube logs (all INFO messages)
1015
1016    # Lines matching a thruputExempt pattern pass every thruput cap and sampling; with one set,
1017    # a spent cap drops the other lines instead of pausing journald
1018
1019    # Move Journald logs reader to a separate process, to prevent process from crashing in case of corrupted log files
1020    spawnExternalProcess = false
1021
1022    # Group each writer's repeated lines into one event per run that counts them.
1023    # Off unless true.
1024    # A writer is a unit, identifier, command, process, priority and facility: never grouped with another.
1025    repeatEnabled = false
1026
1027    # How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
1028    # repeatMaxSpan = 1m
1029
1030    # Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
1031    # repeatOffsets = false
1032
1033    # What the journal, all its writers together, may remember: distinct lines, lines one event stands for, and bytes
1034    # repeatMaxKeys = 256
1035    # repeatMaxCount = 1000
1036    # repeatMaxBytes = 256KB
1037
1038    # Fold each writer's lines of one pattern past foldMaxPerMinute into one summary a minute.
1039    # Lossy: folded lines are not sent. Off unless true.
1040    # A writer is a unit, identifier, command, process, priority and facility: never folded with another.
1041    foldEnabled = false
1042
1043    # Lines of one pattern a minute sent before the rest fold
1044    # foldMaxPerMinute = 60
1045
1046    # Lines never folded (a regular expression; empty: the default)
1047    # foldKeep = (?i)error|exception|fatal|panic
1048
1049    # What the journal, all its writers together, may keep a minute: its patterns and summaries
1050    # foldMaxBytes = 256KB
1051
1052    # Report the journal's top patterns every interval, as events of their own.
1053    # Off unless true.
1054    # A writer is a unit, identifier, command, process, priority and facility: its lines never share a pattern with another's, and the top patterns are of all its writers together.
1055    patternsEnabled = false
1056
1057    # How often a report goes out, from 1m to 24h
1058    # patternsInterval = 15m
1059
1060    # How many patterns a report carries, up to 100
1061    # patternsTop = 20
1062
1063    # What the journal's patterns may keep; past it a new one takes the place of the lightest
1064    # patternsMaxBytes = 256KB
1065
1066    # The report events' type (empty: the input's type with _patterns after it)
1067    # patternsType =
1068
1069    # Pipe to join events (container logs only)
1070    [pipe.join]
1071
1072    disabled = false
1073
1074    maxInterval = 100ms
1075
1076    maxWait = 1s
1077
1078    maxSize = 1MB
1079
1080    # Default pattern to indicate new message (should start not from space)
1081    patternRegex = ^[^\s]
1082
1083    # You can configure global replace rules for the events, which can help to remove sensitive data
1084    # from logs before they are sent to outputs. Those rules will be applied to all pipelines for container logs, host logs,
1085    # application logs and events.
1086    # In the following example we replace password=TEST with password=********
1087    ; [pipe.replace::name]
1088    ; patternRegex = (password=)([^\s]+)
1089    ; replace = $1********
1090
1091    # You can configure global hash rules for the events, which can help to hide sensitive data
1092    # from logs before they are sent to outputs. Those rules will be applied to all pipelines for container logs, host logs,
1093    # application logs and events.
1094    # In the following example we hash IP addresses with fnv-1a-64
1095    ; [pipe.hash::name]
1096    ; match = (\d{1,3}\.){3}\d{1,3}'
1097    ; function = fnv-1a-64
1098
1099    # Log lines matching patternRegex pass every thruput cap and sampling, in the workloads whose
1100    # fields match every matchRegex.<field>; over its cap such a workload drops its other lines
1101    # instead of pausing. Select by namespace: pod, container and image names are the workload
1102    # owner's to choose, and label fields never match
1103    ; [pipe.thruput_exempt::payments-audit]
1104    ; matchRegex.kubernetes_namespace = ^payments$
1105    ; patternRegex = ^AUDIT
1106
1107    # Count the events and bytes leaving every pipeline (pipeline_events, pipeline_bytes);
1108    # maxSeries bounds the label combinations. Fields are written with underscores.
1109    ; [pipe.eventcount]
1110    ; labels = index, sourcetype, output
1111    ; fields = kubernetes_namespace
1112    ; maxSeries = 1000

1113  004-addon.conf: |

1114    [general]
1115    # addons can be run in parallel with agents
1116    addon = true
1117
1118    [input.kubernetes_events]
1119
1120    disabled = false
1121
1122    type = k8s_events
1123
1124    syslog.format = {{type}}|{{timestamp::format(2006-01-02T15:04:05.999999999Z07:00)}}|{{cluster}}|{{message}}
1125
1126    output =
1127
1128    excludeManagedFields = true

1129---
1130apiVersion: apps/v1
1131kind: DaemonSet
1132metadata:
1133  name: collectorforkubernetes-syslog
1134  namespace: collectorforkubernetes-syslog
1135  annotations:
1136    collectord.collectord.io/logs-replace.ignoredebuglog-search: ^(TRACE|DEBUG) .*
1137    collectord.collectord.io/logs-replace.ignoredebuglog-val: ""
1138  labels:
1139    app: collectorforkubernetes-syslog
1140spec:
1141  updateStrategy:
1142    type: RollingUpdate
1143  selector:
1144    matchLabels:
1145      daemon: collectorforkubernetes-syslog
1146  template:
1147    metadata:
1148      name: collectorforkubernetes-syslog
1149      annotations:
1150        # A change to the configuration files these pods mount rolls them:
1151        # Collectord reads its inputs' configuration at startup. A ConfigMap managed
1152        # outside the chart (configMap.create: false) gets no checksum: roll the pods
1153        # with an annotation of your own under podAnnotations.
1154        checksum/config: 57c6ea8d5486414458b15aab87e944a8677184ed61156b171732868f4a28ad6c
1155      labels:
1156        daemon: collectorforkubernetes-syslog
1157    spec:
1158      priorityClassName: "system-node-critical"
1159      dnsPolicy: ClusterFirstWithHostNet
1160      hostNetwork: true
1161      serviceAccountName: collectorforkubernetes-syslog
1162      nodeSelector:
1163        kubernetes.io/os: linux
1164      affinity:
1165        nodeAffinity:
1166          requiredDuringSchedulingIgnoredDuringExecution:
1167            nodeSelectorTerms:
1168            - matchExpressions:
1169              - key: eks.amazonaws.com/compute-type
1170                operator: NotIn
1171                values:
1172                - fargate
1173              - key: type
1174                operator: NotIn
1175                values:
1176                - virtual-kubelet
1177      tolerations:
1178      - effect: NoSchedule
1179        operator: Exists
1180      - effect: NoExecute
1181        operator: Exists
1182      containers:
1183      - name: collectorforkubernetes-syslog
1184        image: registry.outcold.solutions/outcoldsolutions/collectorforkubernetes:26.10.1
1185        imagePullPolicy: Always
1186        securityContext:
1187          runAsUser: 0
1188          privileged: true
1189        resources:
1190          limits:
1191            cpu: 2000m
1192            memory: 1024Mi
1193          requests:
1194            cpu: 500m
1195            memory: 256Mi
1196        env:
1197        - name: KUBERNETES_NODENAME
1198          valueFrom:
1199            fieldRef:
1200              fieldPath: spec.nodeName
1201        - name: KUBERNETES_NODEIP
1202          valueFrom:
1203            fieldRef:
1204              fieldPath: status.hostIP
1205        - name: POD_NAME
1206          valueFrom:
1207            fieldRef:
1208              fieldPath: metadata.name
1209        volumeMounts:
1210        - name: collectorforkubernetes-syslog-state
1211          mountPath: /data
1212        - name: collectorforkubernetes-syslog-config
1213          mountPath: /config/
1214          readOnly: true
1215        - name: rootfs
1216          mountPath: /rootfs/
1217          readOnly: false
1218          mountPropagation: HostToContainer
1219        - name: localtime
1220          mountPath: /etc/localtime
1221          readOnly: true
1222
1223      volumes:
1224      - name: collectorforkubernetes-syslog-state
1225        hostPath:
1226          path: /var/lib/collectorforkubernetes-syslog/data/
1227          type: DirectoryOrCreate
1228      - name: rootfs
1229        hostPath:
1230          path: /
1231      - name: localtime
1232        hostPath:
1233          path: /etc/localtime
1234      - name: collectorforkubernetes-syslog-config
1235        configMap:
1236          name: collectorforkubernetes-syslog
1237          items:
1238          - key: 001-general.conf
1239            path: 001-general.conf
1240          - key: 002-daemonset.conf
1241            path: 002-daemonset.conf
1242---
1243apiVersion: apps/v1
1244kind: Deployment
1245metadata:
1246  name: collectorforkubernetes-syslog-addon
1247  namespace: collectorforkubernetes-syslog
1248  annotations:
1249    collectord.collectord.io/logs-replace.ignoredebuglog-search: ^(TRACE|DEBUG) .*
1250    collectord.collectord.io/logs-replace.ignoredebuglog-val: ""
1251  labels:
1252    app: collectorforkubernetes-syslog
1253spec:
1254  replicas: 1
1255  selector:
1256    matchLabels:
1257      daemon: collectorforkubernetes-syslog
1258  template:
1259    metadata:
1260      name: collectorforkubernetes-syslog-addon
1261      annotations:
1262        # A change to the configuration files these pods mount rolls them:
1263        # Collectord reads its inputs' configuration at startup. A ConfigMap managed
1264        # outside the chart (configMap.create: false) gets no checksum: roll the pods
1265        # with an annotation of your own under podAnnotations.
1266        checksum/config: 1f42d2568dfe2791b5710359d68f76596429e28b71cf85069e740231b975875f
1267      labels:
1268        daemon: collectorforkubernetes-syslog
1269    spec:
1270      priorityClassName: "system-cluster-critical"
1271      serviceAccountName: collectorforkubernetes-syslog
1272      nodeSelector:
1273        kubernetes.io/os: linux
1274      affinity:
1275        nodeAffinity:
1276          requiredDuringSchedulingIgnoredDuringExecution:
1277            nodeSelectorTerms:
1278            - matchExpressions:
1279              - key: eks.amazonaws.com/compute-type
1280                operator: NotIn
1281                values:
1282                - fargate
1283              - key: type
1284                operator: NotIn
1285                values:
1286                - virtual-kubelet
1287      containers:
1288      - name: collectorforkubernetes-syslog
1289        image: registry.outcold.solutions/outcoldsolutions/collectorforkubernetes:26.10.1
1290        imagePullPolicy: Always
1291        securityContext:
1292          allowPrivilegeEscalation: false
1293          capabilities:
1294            drop:
1295            - ALL
1296          privileged: false
1297          runAsUser: 0
1298          seLinuxOptions:
1299            type: spc_t
1300          seccompProfile:
1301            type: RuntimeDefault
1302        resources:
1303          limits:
1304            cpu: 1000m
1305            memory: 512Mi
1306          requests:
1307            cpu: 200m
1308            memory: 64Mi
1309        env:
1310        - name: KUBERNETES_NODENAME
1311          valueFrom:
1312            fieldRef:
1313              fieldPath: spec.nodeName
1314        - name: POD_NAME
1315          valueFrom:
1316            fieldRef:
1317              fieldPath: metadata.name
1318        volumeMounts:
1319        - name: collectorforkubernetes-syslog-state
1320          mountPath: /data
1321        - name: collectorforkubernetes-syslog-config
1322          mountPath: /config/
1323          readOnly: true
1324        - name: localtime
1325          mountPath: /etc/localtime
1326          readOnly: true
1327
1328      volumes:
1329      # The DaemonSet pod on this node owns the directory and the database the
1330      # addon opens read-only. DirectoryOrCreate, so an addon scheduled first
1331      # mounts it too; Collectord waits for the DaemonSet to create the database.
1332      - name: collectorforkubernetes-syslog-state
1333        hostPath:
1334          path: /var/lib/collectorforkubernetes-syslog/data/
1335          type: DirectoryOrCreate
1336      - name: localtime
1337        hostPath:
1338          path: /etc/localtime
1339      - name: collectorforkubernetes-syslog-config
1340        configMap:
1341          name: collectorforkubernetes-syslog
1342          items:
1343          - key: 001-general.conf
1344            path: 001-general.conf
1345          - key: 004-addon.conf
1346            path: 004-addon.conf