Configuration

  1# An upgrade replaces this file: put your settings in 002-user.conf. An environment variable
  2# overrides any value: COLLECTOR__<ANYNAME>=<section>__<key>=<value>, for example
  3#   COLLECTOR__LOGLEVEL=general__logLevel=debug
  4
  5[general]
  6
  7# Please review license https://www.outcoldsolutions.com/legal/license-agreement/
  8# and accept license by changing the value to *true*
  9acceptLicense = false
 10
 11# The read positions and internal state: moving it forwards the logs again. A relative path
 12# resolves against the working directory (/ under systemd), not this file's.
 13dataPath = /var/collectord/
 14
 15# log level (trace, debug, info, warn, error, fatal)
 16logLevel = info
 17
 18# How Collectord spells the fields it composes: _ (the default) or .; the monitoring apps read both
 19# fieldsSeparator = _
 20
 21# host:port of the http server (empty: none), serving /healthz and /readyz, and
 22# /metrics/prometheus and /metrics/json with httpServerEndpoints.metrics = true
 23httpServerBinding =
 24
 25# telemetry report endpoint, set it to empty string to disable telemetry
 26telemetryEndpoint = https://license.outcold.solutions/telemetry/
 27
 28licenseEndpoint = https://license.outcold.solutions/license/
 29licenseServerProxyUrl =
 30
 31# user:password
 32licenseServerProxyBasicAuth =
 33
 34license =
 35
 36# bin/collectord sets HOSTNAME to the kernel's node name (uname -n) when it is not set
 37hostname = ${HOSTNAME}
 38
 39# splunk, or devnull to send nothing
 40defaultOutput = splunk
 41
 42# The index type of every input below that sets no indexType: events, metrics (a Splunk metrics
 43# index) or both, to compare the dashboards before switching
 44defaultIndexType = events
 45
 46fileInputBufferSize = 8kb
 47fileInputLineMaxSize = 1mb
 48
 49# Indexed fields added to every event, named to match ^[a-z][_a-z0-9]*$
 50# ; fields.my_environment = dev
 51fields.linux_cluster = -
 52
 53# EC2 instance metadata as fields: ec2Metadata.<field> = <metadata path>
 54# ec2Metadata.ec2_instance_id = /latest/meta-data/instance-id
 55# ec2Metadata.ec2_instance_type = /latest/meta-data/instance-type
 56
 57# A cap on the logs forwarded a second, such as 512Kb (metrics are not capped); each log input
 58# below takes thruputPerSecond, tooOldEvents and tooNewEvents of its own too
 59thruputPerSecond =
 60
 61# Drop the events older than this, such as 168h, or this far in the future, such as 1h
 62tooOldEvents =
 63tooNewEvents =
 64
 65# With indexType = both, the metrics-index copies go to metricsOutput asynchronously: a slow
 66# metrics output drops copies instead of holding back the events. false holds the pipeline back.
 67; multioutput.async = true
 68# Copies buffered per section; past it and the output's queue they are dropped
 69# (multioutput_dropped_events)
 70; multioutput.asyncBufferSize = 100
 71
 72
 73[license.client]
 74# A license on a web server, or served by a Collectord license server
 75;url = http://10.0.2.2:9199/license
 76;basicAuth = :password
 77insecure = false
 78capath =
 79caname =
 80
 81
 82[input.collectord_metrics]
 83
 84disabled = false
 85type = linux_prometheus
 86interval = 1m
 87output =
 88index =
 89
 90# An indexType here or in the stats and Prometheus inputs below overrides [general]
 91# defaultIndexType. Set metricsIndex or [output.splunk] defaultMetricsIndex: the HEC token's default
 92# index is normally an events index. metricsOutput: an output whose token can write it.
 93metricsIndex =
 94metricsOutput =
 95
 96# Regular expressions on the metric name: with a whitelist, only the matching metrics are sent
 97whitelist.1 = ^file_input_open$
 98whitelist.2 = ^file_input_read_bytes$
 99whitelist.4 = ^pipe$
100whitelist.5 = ^pipelines_num$
101whitelist.6 = ^splunk_post_bytes_sum.*$
102whitelist.7 = ^splunk_post_events_count_sum.*$
103whitelist.8 = ^splunk_post_failed_requests$
104whitelist.9 = ^splunk_post_message_max_lag_seconds_bucket.*$
105whitelist.10 = ^splunk_post_requests_seconds_sum.*$
106whitelist.11 = ^splunk_post_retries_required_sum.*$
107
108# Scrape a Prometheus endpoint (node_exporter, DCGM, an application's /metrics), one
109# [input.prometheus::<name>] section each; the first of endpoint.1, endpoint.2, ... that answers
110# is scraped. podSelector, endpointSlice, service and nodeSelector need Kubernetes: a section with
111# one stops collectord from starting.
112; [input.prometheus::node_exporter]
113; type = linux_prometheus
114; index =
115; host =
116; source = node_exporter
117; interval = 60s
118; endpoint.1 = http://127.0.0.1:9100/metrics
119# token for "Authorization: Bearer $(cat tokenPath)"
120; tokenPath =
121# the CA that signs the server's certificate
122; certPath =
123; clientCertPath =
124; clientKeyPath =
125; insecure = false
126; includeHelp = false
127
128[input.system_stats]
129
130disabled.host = false
131pathCgroups = /sys/fs/cgroup
132pathProc = /proc
133statsInterval = 30s
134type.host = linux_stats_v2_host
135index.host =
136output.host =
137metricsIndex.host =
138metricsOutput.host =
139
140
141[input.mount_stats]
142
143disabled = false
144statsInterval = 30s
145
146# auto: every real filesystem of the host, one event each at its shortest mount point, the mount
147# table reread every interval; root: the root filesystem alone
148mounts = auto
149
150# with mounts = auto: a regular expression the filesystem type must match, and one the mount
151# point must not (empty: the defaults). autofs mounts are never read: it would keep them mounted
152include =
153exclude =
154
155type = linux_mount_stats
156index =
157output =
158metricsIndex =
159metricsOutput =
160
161
162[input.disk_stats]
163
164disabled = false
165statsInterval = 30s
166type = linux_disk_stats
167index =
168metricsIndex =
169metricsOutput =
170output =
171
172[input.proc_stats]
173
174disabled = false
175pathProc = /proc
176statsInterval = 30s
177type = linux_proc_stats_v2
178index.host =
179
180# kernel threads too: often over 100 of them, rarely of use, all indexed
181includeSystemThreads = false
182
183# Each process's systemd unit (systemd_unit), as journal entries name it; every container is a
184# scope of its own, a new value per container
185systemdUnit = true
186
187output.host =
188metricsIndex =
189metricsOutput =
190
191
192[input.net_stats]
193
194disabled = false
195pathProc = /proc
196statsInterval = 30s
197type = linux_net_stats_v2
198index =
199output =
200metricsIndex =
201metricsOutput =
202
203
204[input.net_socket_table]
205
206disabled = false
207pathProc = /proc
208statsInterval = 30s
209type = linux_net_socket_table
210index =
211output =
212
213# one event per group of connections sharing a state and address pair, not one per connection
214group = true
215
216
217# The failed systemd units: one event per interval, also when none failed
218[input.systemd_units]
219
220disabled = false
221interval = 60s
222systemctl = systemctl
223timeout = 10s
224type = linux_systemd_units
225index =
226output =
227
228
229# Pending package updates from apt's or dnf's cache (no network, no lock): one event per
230# interval, also when none are pending; yum and zypper report "unsupported". Where there is no
231# nobody user, set user = (empty) to run it as root; with the tarball extracted elsewhere, point
232# command there.
233[input.script::package_updates]
234
235disabled = false
236command = /opt/collectorforlinux/scripts/package-updates.sh
237interval = 1h
238timeout = 2m
239user = nobody
240type = linux_package_updates
241source = package-updates
242index =
243output =
244
245
246# Run an executable every interval and send what it prints, one [input.script::<name>] section
247# each. It runs without a shell, as root unless user says, with a fixed environment (PATH,
248# LANG=C, HOME, the COLLECTORD_* variables, then env.<n>), never Collectord's, and only while it
249# and every directory above it are owned by root and writable by no one else. A change to these
250# sections takes a restart.
251; [input.script::uptime]
252; disabled = false
253; command = /opt/scripts/uptime.sh
254; arg.1 = --format
255; arg.2 = json
256; env.1 = TZ=UTC
257; interval = 5m
258; startDelay = 0s
259# runs never overlap, so it may be longer than interval
260; timeout = 60s
261# event: all of the output is one event; lines: one event per line; prometheus: see below
262; format = event
263# with format = lines, a line this does not match continues the event before it
264; eventPattern =
265# a name or uid, optionally :group or :gid; empty runs as root
266; user = nobody
267; workingDirectory = /
268# a run printing more is stopped: format = lines sends the complete lines before it, the others nothing
269; maxOutputSize = 1MB
270# the events' sourcetype (default is the section's name) and source (default is command)
271; type = linux_uptime
272; source =
273; index =
274; output =
275
276# With format = prometheus a script prints a Prometheus text exposition (what node_exporter's
277# textfile collector reads): a sample is an event, or with indexType = metrics or both a
278# metrics-index record; a counter's event carries its rise since the last run.
279; [input.script::backup_metrics]
280; disabled = false
281; command = /opt/scripts/backup-metrics.sh
282; interval = 1m
283; user = nobody
284; format = prometheus
285; indexType = metrics
286; metricsIndex =
287; metricsOutput =
288; whitelist.1 = ^backup_
289; blacklist.1 =
290; includeHelp = false
291; type = linux_backup
292
293
294# syslog, messages and auth.log. Off: rsyslog writes them from the journal [input.journald] reads,
295# so every line would arrive twice. On a host without journald, turn this on and journald off.
296[input.files::syslog]
297
298disabled = true
299path = /var/log/
300match = ^(syslog|messages|auth\.log)(.[\d\-]+)?$
301recursive = false
302
303# how often a file read to its end is checked for more
304pollingInterval = 250ms
305
306# how often path is searched for new files
307walkingInterval = 5s
308
309# add the file offset to the events
310verboseFields = false
311
312type = linux_host_logs
313index =
314extraction = ^(?P<timestamp>[A-Za-z]+\s+\d+\s\d+:\d+:\d+|\d{4}-\d{2}-\d{2}T\S+)\s(?P<syslog_hostname>[^\s]+)\s(?P<syslog_component>[^:\[]+)(\[(?P<syslog_pid>\d+)\])?: (.+)$
315
316# the extracted fields naming the process that wrote a line, which groups its lines apart
317extractionWriterFields = syslog_hostname, syslog_component, syslog_pid
318
319timestampField = timestamp
320
321# a Go layout of the reference time `Mon Jan 2 15:04:05 -0700 MST 2006`; @syslog reads RFC 3339
322# and the traditional `Jan 2 15:04:05`
323timestampFormat = @syslog
324
325# take the month and day from today when the format has none
326timestampSetMonth = false
327timestampSetDay = false
328
329# the time zone when the format has none
330timestampLocation = Local
331
332# the percent of lines forwarded: 20 sends 20%, -1 all of them
333samplingPercent = -1
334
335# hash-based sampling: a regular expression with a named group `key`
336samplingKey =
337
338output =
339thruputPerSecond =
340tooOldEvents =
341tooNewEvents =
342
343# a line this does not match continues the event before it
344eventPattern = ^[^\s]
345
346# whitelist.0 = ^regexp$
347# blacklist.0 = ^regexp$
348
349# Lines matching thruputExempt.<n> pass every thruput cap and sampling; with one set, a spent
350# cap drops the other lines instead of pausing this input
351# thruputExempt.0 = ^regexp$
352
353# Group each file's repeated lines into one event per run that counts them. Off unless true.
354repeatEnabled = false
355# How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
356# repeatMaxSpan = 1m
357# Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
358# repeatOffsets = false
359# What each stream may remember: distinct lines, lines one event stands for, and bytes
360# repeatMaxKeys = 256
361# repeatMaxCount = 1000
362# repeatMaxBytes = 256KB
363
364# Fold each file's lines of one pattern past foldMaxPerMinute into one summary a minute.
365# Lossy: folded lines are not sent. Off unless true.
366foldEnabled = false
367# Lines of one pattern a minute sent before the rest fold
368# foldMaxPerMinute = 60
369# Lines never folded (a regular expression; empty: the default)
370# foldKeep = (?i)error|exception|fatal|panic
371# What each file may keep a minute: its patterns and summaries
372# foldMaxBytes = 256KB
373
374# Report each file's top patterns every interval, as events of their own. Off unless true.
375patternsEnabled = false
376# How often a report goes out, from 1m to 24h
377# patternsInterval = 15m
378# How many patterns a report carries, up to 100
379# patternsTop = 20
380# What each file's patterns may keep; past it a new one takes the place of the lightest
381# patternsMaxBytes = 256KB
382# The report events' type (empty: this section's type with _patterns after it)
383# patternsType =
384
385
386[input.files::logs]
387
388disabled = false
389path = /var/log/
390match = ^(([\w\-.]+\.log(.[\d\-]+)?)|(docker))$
391
392# the files rsyslog writes from the journal on Debian and Ubuntu, which [input.journald] reads
393exclude = ^(auth|cron|daemon|kern|lpr|mail|user|ufw)\.log(.[\d\-]+)?$
394
395pollingInterval = 250ms
396walkingInterval = 5s
397verboseFields = false
398type = linux_host_logs
399index =
400extraction =
401timestampField =
402
403# a Go layout of the reference time `Mon Jan 2 15:04:05 -0700 MST 2006`
404timestampFormat =
405
406# the time zone when the format has none
407timestampLocation =
408
409samplingPercent = -1
410samplingKey =
411output =
412thruputPerSecond =
413tooOldEvents =
414tooNewEvents =
415eventPattern = ^[^\s]
416
417# whitelist.0 = ^regexp$
418# blacklist.0 = ^regexp$
419
420# Lines matching thruputExempt.<n> pass every thruput cap and sampling; with one set, a spent
421# cap drops the other lines instead of pausing this input
422# thruputExempt.0 = ^regexp$
423
424# Group each file's repeated lines into one event per run that counts them. Off unless true.
425repeatEnabled = false
426# How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
427# repeatMaxSpan = 1m
428# Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
429# repeatOffsets = false
430# What each stream may remember: distinct lines, lines one event stands for, and bytes
431# repeatMaxKeys = 256
432# repeatMaxCount = 1000
433# repeatMaxBytes = 256KB
434
435# Fold each file's lines of one pattern past foldMaxPerMinute into one summary a minute.
436# Lossy: folded lines are not sent. Off unless true.
437foldEnabled = false
438# Lines of one pattern a minute sent before the rest fold
439# foldMaxPerMinute = 60
440# Lines never folded (a regular expression; empty: the default)
441# foldKeep = (?i)error|exception|fatal|panic
442# What each file may keep a minute: its patterns and summaries
443# foldMaxBytes = 256KB
444
445# Report each file's top patterns every interval, as events of their own. Off unless true.
446patternsEnabled = false
447# How often a report goes out, from 1m to 24h
448# patternsInterval = 15m
449# How many patterns a report carries, up to 100
450# patternsTop = 20
451# What each file's patterns may keep; past it a new one takes the place of the lightest
452# patternsMaxBytes = 256KB
453# The report events' type (empty: this section's type with _patterns after it)
454# patternsType =
455
456[input.journald]
457
458disabled = false
459path.persistent = /var/log/journal/
460path.volatile = /run/log/journal/
461
462# how often the journal read to its end is checked for more
463pollingInterval = 250ms
464
465type = linux_host_logs
466index =
467samplingPercent = -1
468samplingKey =
469
470# reopen the journal this often, to let go of the files journald deleted
471reopenInterval = 1h
472
473output =
474thruputPerSecond =
475tooOldEvents =
476tooNewEvents =
477eventPattern = ^[^\s]
478
479# whitelist.0 = ^regexp$
480# blacklist.0 = ^regexp$
481
482# Lines matching thruputExempt.<n> pass every thruput cap and sampling; with one set, a spent
483# cap drops the other lines instead of pausing this input
484# thruputExempt.0 = ^regexp$
485
486# Group each writer's repeated lines (unit, identifier, command, process, priority and facility
487# tell writers apart) into one event per run that counts them. Off unless true.
488repeatEnabled = false
489# How long a run is held before its event goes out, up to 10m (a crash resends up to that much)
490# repeatMaxSpan = 1m
491# Add every line's time to a run's event (collectord_repeat_offsets, milliseconds from its own)
492# repeatOffsets = false
493# What the journal, all its writers together, may remember: distinct lines, lines one event stands for, and bytes
494# repeatMaxKeys = 256
495# repeatMaxCount = 1000
496# repeatMaxBytes = 256KB
497
498# Fold each writer's lines of one pattern past foldMaxPerMinute into one summary a minute.
499# Lossy: folded lines are not sent. Off unless true.
500foldEnabled = false
501# Lines of one pattern a minute sent before the rest fold
502# foldMaxPerMinute = 60
503# Lines never folded (a regular expression; empty: the default)
504# foldKeep = (?i)error|exception|fatal|panic
505# What the journal, all its writers together, may keep a minute: its patterns and summaries
506# foldMaxBytes = 256KB
507
508# Report the journal's top patterns, of all its writers together, every interval, as events of
509# their own. Off unless true.
510patternsEnabled = false
511# How often a report goes out, from 1m to 24h
512# patternsInterval = 15m
513# How many patterns a report carries, up to 100
514# patternsTop = 20
515# What the journal's patterns may keep; past it a new one takes the place of the lightest
516# patternsMaxBytes = 256KB
517# The report events' type (empty: this section's type with _patterns after it)
518# patternsType =
519
520[pipe.join]
521
522maxInterval = 100ms
523maxWait = 1s
524maxSize = 100K
525
526# Measure maxInterval and maxWait from the last line instead of the event's first: a source
527# printing a line just under the gap then holds its event (and its acknowledgement) until maxSize
528gapBased = false
529
530# Replace rules for the logs and events (not the stats or Prometheus samples), to remove sensitive
531# data before it is sent; this one masks password=TEST as password=********
532; [pipe.replace::name]
533; patternRegex = (password=)([^\s]+)
534; replace = $1********
535
536# Count the events and bytes leaving every pipeline (pipeline_events, pipeline_bytes) by these
537# labels; maxSeries bounds the combinations. Add both names to the whitelist above to forward them.
538; [pipe.eventcount]
539; labels = index, sourcetype, output
540; fields =
541; maxSeries = 1000
542
543[output.splunk]
544
545url =
546# several HEC URLs, all with the same path:
547# urls.0 = https://server1:8088/services/collector/event/1.0
548# urls.1 = https://server1:8088/services/collector/event/1.0
549# urls.2 = https://server1:8088/services/collector/event/1.0
550
551# how a URL is picked from several, at first and after each failure (connection or HTTP 5xx):
552# urlSelection = random|round-robin|random-with-round-robin
553# random picks at random, at first and after each failure; random-with-round-robin and round-robin
554# take the next URL after a failure, so the instances on a failed URL can all move to the same one
555urlSelection = random
556
557token =
558insecure = false
559caPath =
560caName =
561clientCertPath =
562clientKeyPath =
563
564# a batch goes out at batchSize, at events, or after frequency
565frequency = 5s
566batchSize = 768K
567# 0: no limit
568events = 50
569
570proxyUrl =
571
572# user:password
573proxyBasicAuth =
574
575# Splunk acknowledgement url (.../services/collector/ack)
576ackUrl =
577# with several urls, ackUrls.<n> in the same order, so a payload is acknowledged where it went:
578# ackUrls.0 = https://server1:8088/services/collector/ack
579# ackUrls.1 = https://server1:8088/services/collector/ack
580# ackUrls.2 = https://server1:8088/services/collector/ack
581
582ackEnabled = false
583ackTimeout = 3m
584
585# per request, connecting and reading the response included
586timeout = 30s
587
588# a client per index, so events with an incorrect index do not block the other pipelines
589dedicatedClientPerIndex = true
590
591# RedirectToDefault, Drop or Retry
592incorrectIndexBehavior = RedirectToDefault
593
594# gzip compression level (nocompression, default, 1...9)
595compressionLevel = default
596
597# more threads for over 4k events a second
598threads = 1
599
600# roundrobin, or weighted (beta)
601threadsAlgorithm = weighted
602
603# Most connections to one HEC host (unset: 8 per thread; 0: no limit); not applied through a
604# SOCKS5 proxyUrl
605; maxConnsPerHost = 8
606
607# indexed fields to leave out of the events
608# excludeFields.kubernetes_pod_ip = true
609
610# With fieldsSeparator = ., also send every built-in field under its underscore name (events only)
611# duplicateFieldsWithUnderscore = false
612
613# Drop the events that name no index, instead of sending them to the HEC token's default index
614; requireExplicitIndex = true
615
616# The metrics index of every input with indexType = metrics or both and no metricsIndex of its
617# own; unset, the HEC token's default index
618; defaultMetricsIndex = linux_metrics
619
620# Truncate messages longer than this
621; maximumMessageLength = 1M
622
623# Events queued for the output: more rides out longer network trouble, at the cost of memory
624queueSize = 1024
625
626# Digits of the timestamps' fraction: 3, milliseconds (the default); 6, microseconds; 9, nanoseconds
627; timestampPrecision = 3
628
629# Report low entropy
630; [diagnostics::node-entropy]
631; settings.path = /proc/sys/kernel/random/entropy_avail
632; settings.interval = 1h
633; settings.threshold = 800
634
635# Report a pending reboot
636[diagnostics::node-reboot-required]
637settings.path = /var/run/reboot-required*
638settings.interval = 1h