1# collectord configuration file
  2#
  3# Run collectord with flag -conf and specify location of the configuration file.
  4#
  5# You can override all the values using environment variables with the format like
  6#   COLLECTOR__<ANYNAME>=<section>__<key>=<value>
  7# As an example you can set dataPath in [general] section as
  8#   COLLECTOR__DATAPATH=general__dataPath=C:\\some\\path\\data.db
  9# This parameter can be configured using -env-override, set it to empty string to disable this feature
 10
 11[general]
 12
 13# (obsolete, use acceptLicense instead)
 14# acceptEULA = false
 15
 16# Please review license https://www.outcoldsolutions.com/docs/license-agreement/
 17# and accept license by changing the value to *true*
 18acceptLicense = false
 19
 20# location for the database
 21# is used to store position of the files and internal state
 22dataPath = ../var/collectord
 23
 24# log level (trace, debug, info, warn, error, fatal)
 25logLevel = info
 26
 27# http server gives access to two endpoints
 28# /healthz
 29# /metrics
 30httpServerBinding =
 31
 32# telemetry report endpoint, set it to empty string to disable telemetry
 33telemetryEndpoint = https://license.outcold.solutions/telemetry/
 34
 35# license check endpoint
 36licenseEndpoint = https://license.outcold.solutions/license/
 37
 38# license server through proxy
 39licenseServerProxyUrl =
 40
 41# authentication with basic authorization (user:password)
 42licenseServerProxyBasicAuth =
 43
 44# license key
 45license =
 46
 47# docker daemon hostname is used by default as hostname
 48# use this configuration to override
 49hostname = ${HOSTNAME}
 50
 51# Default output for events, logs and metrics
 52# valid values: splunk and devnull
 53# Use devnull by default if you don't want to redirect data
 54defaultOutput = splunk
 55
 56# Default buffer size for file input
 57fileInputBufferSize = 256b
 58
 59# Maximum size of one line the file reader can read
 60fileInputLineMaxSize = 1mb
 61
 62# Include custom fields to attach to every event, in example below every event sent to Splunk will hav
 63# indexed field my_environment=dev. Fields names should match to ^[a-z][_a-z0-9]*$
 64# Better way to configure that is to specify labels for Docker Hosts.
 65# ; fields.my_environment = dev
 66fields.linux_cluster = -
 67
 68# Include EC2 Metadata (see list of possible fields https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
 69# Should be in format ec2Metadata.{desired_field_name} = {url path to read the value}
 70# ec2Metadata.ec2_instance_id = /latest/meta-data/instance-id
 71# ec2Metadata.ec2_instance_type = /latest/meta-data/instance-type
 72
 73# subdomain for the annotations added to the pods, workloads, namespaces or containers, like splunk.collectord.io/..
 74annotationsSubdomain =
 75
 76# Configure acknowledgement database.
 77# - force fsync on every write to Write-Ahead-Log
 78db.fsync = false
 79# - maximum size of the Write-Ahead-Log
 80db.compactAt = 1M
 81
 82# configure global thruput per second for forwarded logs (metrics are not included)
 83# for example if you set `thruputPerSecond = 512Kb`, that will limit amount of logs forwarded
 84# from the single Collectord instance to 512Kb per second.
 85# You can configure thruput individually for the logs (including specific for container logs) below
 86thruputPerSecond =
 87
 88# Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
 89# older than 7 days
 90tooOldEvents =
 91
 92# Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
 93tooNewEvents =
 94
 95
 96# cgroup input
 97# sends stas for the host and cgroups (containers)
 98[input.system_stats]
 99
100# disable system level stats
101disabled.host = false
102
103# cgroups fs location
104pathCgroups = /sys/fs/cgroup
105
106# proc location
107pathProc = /proc
108
109# how often to collect cgroup stats
110statsInterval = 30s
111
112# override type
113type.host = linux_stats_v2_host
114
115# specify Splunk index
116index.host =
117
118# set output (splunk or devnull, default is [general]defaultOutput)
119output.host =
120
121
122# mount input (collects mount stats where docker runtime is stored)
123[input.mount_stats]
124
125# disable system level stats
126disabled = false
127
128# how often to collect mount stats
129statsInterval = 30s
130
131# override type
132type = linux_mount_stats
133
134# specify Splunk index
135index =
136
137# set output (splunk or devnull, default is [general]defaultOutput)
138output =
139
140
141# proc input
142[input.proc_stats]
143
144# disable proc level stats
145disabled = false
146
147# proc location
148pathProc = /proc
149
150# how often to collect proc stats
151statsInterval = 60s
152
153# override type
154type = linux_proc_stats_v2
155
156# specify Splunk index
157index.host =
158
159# proc filesystem includes by default system threads (there can be over 100 of them)
160# these stats do not help with the observability
161# excluding them can reduce the size of the index, performance of the searches and usage of the collector
162includeSystemThreads = false
163
164# set output (splunk or devnull, default is [general]defaultOutput)
165output.host =
166
167
168# network stats
169[input.net_stats]
170
171# disable net stats
172disabled = false
173
174# proc path location
175pathProc = /proc
176
177# how often to collect net stats
178statsInterval = 30s
179
180# override type
181type = linux_net_stats_v2
182
183# specify Splunk index
184index =
185
186# set output (splunk or devnull, default is [general]defaultOutput)
187output =
188
189
190# network socket table
191[input.net_socket_table]
192
193# disable net stats
194disabled = false
195
196# proc path location
197pathProc = /proc
198
199# how often to collect net stats
200statsInterval = 30s
201
202# override type
203type = linux_net_socket_table
204
205# specify Splunk index
206index =
207
208# set output (splunk or devnull, default is [general]defaultOutput)
209output =
210
211# group connections by tcp_state, localAddr, remoteAddr (if localPort is not the port it is listening on)
212# that can significally reduces the amount of events
213group = true
214
215
216# Input syslog(.\d+)? files
217[input.files::syslog]
218
219# disable host level logs
220disabled = false
221
222# root location of log files
223path = /var/log/
224
225# regex matching pattern
226match = ^(syslog|messages)(.\d+)?$
227
228# limit search only on one level
229recursive = false
230
231# files are read using polling schema, when reach the EOF how often to check if files got updated
232pollingInterval = 250ms
233
234# how often o look for the new files under logs path
235walkingInterval = 5s
236
237# include verbose fields in events (file offset)
238verboseFields = false
239
240# override type
241type = linux_host_logs
242
243# specify Splunk index
244index =
245
246# field extraction
247extraction = ^(?P<timestamp>[A-Za-z]+\s+\d+\s\d+:\d+:\d+)\s(?P<syslog_hostname>[^\s]+)\s(?P<syslog_component>[^:\[]+)(\[(?P<syslog_pid>\d+)\])?: (.+)$
248
249# timestamp field
250timestampField = timestamp
251
252# format for timestamp
253# the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006`
254timestampFormat = Jan 2 15:04:05
255
256# Adjust date, if month/day aren't set in format
257timestampSetMonth = false
258timestampSetDay = false
259
260# timestamp location (if not defined by format)
261timestampLocation = Local
262
263# sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
264samplingPercent = -1
265
266# sampling key for hash based sampling (should be regexp with the named match pattern `key`)
267samplingKey =
268
269# set output (splunk or devnull, default is [general]defaultOutput)
270output =
271
272# configure default thruput per second for for each container log
273# for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
274# from the single container to 128Kb per second.
275thruputPerSecond =
276
277# Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
278# older than 7 days
279tooOldEvents =
280
281# Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
282tooNewEvents =
283
284
285# Input all *.log(.\d+)? files
286[input.files::logs]
287
288# disable host level logs
289disabled = false
290
291# root location of log files
292path = /var/log/
293
294# regex matching pattern
295match = ^(([\w\-.]+\.log(.[\d\-]+)?)|(docker))$
296
297# files are read using polling schema, when reach the EOF how often to check if files got updated
298pollingInterval = 250ms
299
300# how often o look for the new files under logs path
301walkingInterval = 5s
302
303# include verbose fields in events (file offset)
304verboseFields = false
305
306# override type
307type = linux_host_logs
308
309# specify Splunk index
310index =
311
312# field extraction
313extraction =
314
315# timestamp field
316timestampField =
317
318# format for timestamp
319# the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006`
320timestampFormat =
321
322# timestamp location (if not defined by format)
323timestampLocation =
324
325# sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
326samplingPercent = -1
327
328# sampling key (should be regexp with the named match pattern `key`)
329samplingKey =
330
331# set output (splunk or devnull, default is [general]defaultOutput)
332output =
333
334# configure default thruput per second for for each container log
335# for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
336# from the single container to 128Kb per second.
337thruputPerSecond =
338
339# Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
340# older than 7 days
341tooOldEvents =
342
343# Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
344tooNewEvents =
345
346
347[input.journald]
348
349# disable host level logs
350disabled = false
351
352# root location of log files
353path.persistent = /var/log/journal/
354path.volatile = /run/log/journal/
355
356# when reach end of journald, how often to pull
357pollingInterval = 250ms
358
359# override type
360type = linux_host_logs
361
362# specify Splunk index
363index =
364
365# sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
366samplingPercent = -1
367
368# sampling key (should be regexp with the named match pattern `key`)
369samplingKey =
370
371# how often to reopen the journald to free old files
372reopenInterval = 1h
373
374# set output (splunk or devnull, default is [general]defaultOutput)
375output =
376
377# configure default thruput per second for for each container log
378# for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
379# from the single container to 128Kb per second.
380thruputPerSecond =
381
382# Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
383# older than 7 days
384tooOldEvents =
385
386# Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
387tooNewEvents =
388
389
390# Default configuration for join multi-lines
391[pipe.join]
392
393# Maximum interval of messages in pipeline
394maxInterval = 100ms
395
396# Maximum time to wait for the messages in pipeline
397maxWait = 1s
398
399# Maximum message size
400maxSize = 100K
401
402
403# Splunk output
404[output.splunk]
405
406# Splunk HTTP Event Collector url
407url =
408# You can specify muiltiple splunk URls with
409#
410# urls.0 = https://server1:8088/services/collector/event/1.0
411# urls.1 = https://server1:8088/services/collector/event/1.0
412# urls.2 = https://server1:8088/services/collector/event/1.0
413#
414# Limitations:
415# * The urls cannot have different path.
416
417# Specify how URL should be picked up (in case if multiple is used)
418# urlSelection = random|round-robin|random-with-round-robin
419# where:
420# * random - choose random url on first selection and after each failure (connection or HTTP status code >= 500)
421# * round-robin - choose url starting from first one and bump on each failure (connection or HTTP status code >= 500)
422# * random-with-round-robin - choose random url on first selection and after that in round-robin on each
423#                             failure (connection or HTTP status code >= 500)
424urlSelection = random-with-round-robin
425
426# Splunk HTTP Event Collector Token
427token =
428
429# Allow invalid SSL server certificate
430insecure = false
431
432# Path to CA cerificate
433caPath =
434
435# CA Name to verify
436caName =
437
438# path for client certificate (if required)
439clientCertPath =
440
441# path for client key (if required)
442clientKeyPath =
443
444# Events are batched with the maximum size set by batchSize and staying in pipeline for not longer
445# than set by frequency
446frequency = 5s
447batchSize = 768K
448# limit by the number of events (0 value has no limit on the number of events)
449events = 50
450
451# Splunk through proxy
452proxyUrl =
453
454# authentication with basic authorization (user:password)
455proxyBasicAuth =
456
457# Splunk acknowledgement url (.../services/collector/ack)
458ackUrl =
459# You can specify muiltiple splunk URls for ackUrl
460#
461# ackUrls.0 = https://server1:8088/services/collector/ack
462# ackUrls.1 = https://server1:8088/services/collector/ack
463# ackUrls.2 = https://server1:8088/services/collector/ack
464#
465# Make sure that they in the same order as urls for url, to make sure that this Splunk instance will be
466# able to acknowledge the payload.
467#
468# Limitations:
469# * The urls cannot have different path.
470
471# Enable index acknowledgment
472ackEnabled = false
473
474# Index acknowledgment timeout
475ackTimeout = 3m
476
477# Timeout specifies a time limit for requests made by collectord.
478# The timeout includes connection time, any
479# redirects, and reading the response body.
480timeout = 30s
481
482# in case when pipeline can post to multiple indexes, we want to avoid posibility of blocking
483# all pipelines, because just some events have incorrect index
484dedicatedClientPerIndex = true
485
486# (obsolete) in case if some indexes aren't used anymore, how often to destroy the dedicated client
487# dedicatedClientCleanPeriod = 24h
488
489# possible values: RedirectToDefault, Drop, Retry
490incorrectIndexBehavior = RedirectToDefault
491
492# gzip compression level (nocompression, default, 1...9)
493compressionLevel = default
494
495# number of dedicated splunk output threads (to increase throughput above 4k events per second)
496threads = 1