apiVersion: v1 kind: ServiceAccount metadata: labels: app: collectorforkubernetes name: collectorforkubernetes namespace: default --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app: collectorforkubernetes name: collectorforkubernetes rules: - apiGroups: - "" - apps - batch - extensions - monitoring.coreos.com resources: - namespaces - events - cronjobs - daemonsets - deployments - jobs - nodes - pods - replicasets - replicationcontrollers - scheduledjobs - statefulsets - prometheuses verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: app: collectorforkubernetes name: collectorforkubernetes roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: collectorforkubernetes subjects: - kind: ServiceAccount name: collectorforkubernetes namespace: default --- apiVersion: v1 kind: ConfigMap metadata: name: collectorforkubernetes labels: app: collectorforkubernetes data: collector.conf: | # collector configuration file # # Run collector with flag -conf and specify location of the configuration file. # # You can override all the values using environment variables with the format like # COLLECTOR__=
__= # As an example you can set dataPath in [general] section as # COLLECTOR__DATAPATH=general__dataPath=C:\\some\\path\\data.db # This parameter can be configured using -env-override, set it to empty string to disable this feature [general] # Please review EULA https://www.outcoldsolutions.com/docs/license-agreement/ # and accept eula by uncommenting this code and changing value to *true* ; acceptEULA = false # location for the database # is used to store position of the files and internal state ; dataPath = ./data/ # log level (trace, debug, info, warn, error, fatal) ; logLevel = info # http server gives access to two endpoints # /healthz # /metrics ; httpServerBinding = :8080 # telemetry report endpoint, set it to empty string to disable telemetry ; telemetryEndpoint = https://license.outcold.solutions/telemetry/ # license check endpoint ; licenseEndpoint = https://license.outcold.solutions/license/ # license server through proxy ; licenseServerProxyUrl = # license ; license = # docker daemon hostname is used by default as hostname # use this configuration to override ; hostname = # Include custom fields to attach to every event, in example below every event sent to Splunk will hav # indexed field my_environment=dev. Fields names should match to ^[a-z][_a-z0-9]*$ # Better way to configure that is to specify labels for Kubernetes Nodes. # ; fields.my_environment = dev # connection to docker host [general.docker] # url for docker API, only unix socket is supported ; url = unix:///rootfs/var/run/docker.sock # path to docker root folder (can fallback to use folder structure to read docker metadata) ; dockerRootFolder = /rootfs/var/lib/docker/ # In case if pod metadata was not retrievied. how often collector should retry to reload the pod metadata ; MetadataFetchRetry = 1s # In case if event is recent, how long pipeline should wait for the metadata to be available in Kubernetes API ; MetadataFetchWait = 5s # In case if collector does not see new events for specific container and with the last metadata refresh # We have not found this container - fow how long we should keep this metadata in cache. ; MetadataTTL = 5m # connection to kubernetes host [general.kubernetes] # name of kubernetes node (required) ; nodeName = ; tokenPath = /var/run/secrets/kubernetes.io/serviceaccount/token ; certPath = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt # In case if pod metadata was not retrievied. how often collector should retry to reload the pod metadata ; MetadataFetchRetry = 5s # In case if event is recent, how long pipeline should wait for the metadata to be available in Kubernetes API ; MetadataFetchWait = 30s # In case if collector does not see new events for specific container and with the last metadata refresh # We have not found this container - fow how long we should keep this metadata in cache. ; MetadataTTL = 5m # cgroup input [input.system_stats] # disable system level stats ; disabled = false # cgroups fs location ; pathCgroups = /rootfs/sys/fs/cgroup # proc location ; pathProc = /rootfs/proc # how often to collect cgroup stats ; statsInterval = 30s # override type ; type = kubernetes_stats # specify Splunk index ; index = # proc input [input.proc_stats] # disable proc level stats ; disabled = false # proc location ; pathProc = /rootfs/proc # how often to collect proc stats ; statsInterval = 30s # override type ; type = kubernetes_proc_stats # specify Splunk index ; index = # Log files [input.files] # disable container logs monitoring ; disabled = false # root location of docker files ; path = /rootfs/var/lib/docker/containers/ # glob matching pattern for log files ; glob = */*-json.log* # files are read using polling schema, when reach the EOF how often to check if files got updated ; pollingInterval = 250ms # how often to look for the new files under logs path ; walkingInterval = 5s # include verbose fields in events (file offset) ; verboseFields = false # override type ; type = kubernetes_logs # specify Splunk index ; index = # Input syslog(.\d+)? files [input.files::syslog] # disable host level logs ; disabled = false # root location of docker files path = /rootfs/var/log/ # regex matching pattern match = ^(syslog|messages)(.\d+)?$ # limit search only on one level recursive = false # files are read using polling schema, when reach the EOF how often to check if files got updated pollingInterval = 250ms # how often o look for the new files under logs path walkingInterval = 5s # include verbose fields in events (file offset) verboseFields = false # override type type = kubernetes_host_logs # specify Splunk index ; index = # field extraction extraction = ^(?P[A-Za-z]+\s+\d+\s\d+:\d+:\d+)\s(?P[^\s]+)\s(?P[^:\[]+)(\[(?P\d+)\])?: (.+)$ # timestamp field timestampField = timestamp # format for timestamp # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006` timestampFormat = Jan 2 15:04:05 # Adjust date, if month/day aren't set in format ; timestampSetMonth = false ; timestampSetDay = false # timestamp location (if not defined by format) timestampLocation = Local # Input all *.log(.\d+)? files [input.files::logs] # disable host level logs ; disabled = false # root location of docker files path = /rootfs/var/log/ # regex matching pattern match = ^[\w]+\.log(.\d+)?$ # files are read using polling schema, when reach the EOF how often to check if files got updated pollingInterval = 250ms # how often o look for the new files under logs path walkingInterval = 5s # include verbose fields in events (file offset) verboseFields = false # override type type = kubernetes_host_logs # specify Splunk index ; index = # field extraction ; extraction = # timestamp field ; timestampField = # format for timestamp # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006` ; timestampFormat = # timestamp location (if not defined by format) ; timestampLocation = [input.kubernetes_events] # disable host level logs ; disabled = false # override type ; type = kubernetes_events # specify Splunk index ; index = # Set the timeout for how long request to watch events going to hang reading. ; eventsWatchTimeout = 30m # Ignore events last seen later that this duration. ; eventsTTL = 12h # Splunk output [output.splunk] # Splunk HTTP Event Collector url ; url = # Splunk HTTP Event Collector Token ; token = # Allow invalid SSL server certificate ; insecure = false # Path to CA cerificate ; caPath = # CA Name to verify ; caName = # Events are batched with the maximum size set by batchSize and staying in pipeline for not longer # than set by frequency ; frequency = 5s ; batchSize = 768K # Splunk through proxy ; proxyUrl = # Splunk acknowledgement url (.../services/collector/ack) ; ackUrl = # Enable index acknowledgment ; ackEnabled = false # Index acknowledgment timeout ; ackTimeout = 3m # Pipe to join events (container logs only) [pipe.join] # disable joining event ; disabled = false # Maximum interval of messages in pipeline ; maxInterval = 100ms # Maximum time to wait for the messages in pipeline ; maxWait = 1s # Maximum message size ; maxSize = 100K # Default pattern to indicate new message (should start not from space) ; patternRegex = ^[^\s] # Kube API Server has trace messages with multi line events [pipe.join::kube-apiserver] ; disabled = false matchRegex.kubernetes_container_image = ^gcr.io/google_containers/kube-apiserver-.*$ matchRegex.docker_stream = stderr patternRegex = ^[IWEF]\d{4}\s\d{2}:\d{2}:\d{2}.\d{6}\s # Define special event join patterns for matched events # Section consist of [pipe.join::] # [pipe.join::my_app] ## Set match pattern for the fields #; matchRegex.docker_container_image = my_app #; matchRegex.docker_stream = stdout ## All events start from '[' #; patternRegex = ^\[\d+ --- apiVersion: apps/v1 kind: DaemonSet metadata: name: collectorforkubernetes labels: app: collectorforkubernetes spec: # Default updateStrategy is OnDelete. For collector RollingUpdate is suitable # When you update configuration updateStrategy: type: RollingUpdate selector: matchLabels: daemon: collectorforkubernetes template: metadata: name: collectorforkubernetes labels: daemon: collectorforkubernetes spec: # Service account we use to connect to kubernetes proxy serviceAccountName: collectorforkubernetes # This allows to run DaemonSet pods on Master (or all nodes, which tained with NoSchedule) tolerations: - operator: "Exists" effect: "NoSchedule" - operator: "Exists" effect: "NoExecute" containers: - name: collectorforkubernetes # Collector version image: outcoldsolutions/collectorforkubernetes:3.0.87.180322 securityContext: # Privileged only required to get access to IO in /proc file system. # You can disable privileged and still get most of the metrics and logs. # # Another option can be with enabling just SYS_PTRACE, that will allow access to most # processes io statistics (excluded processes running outside of containers) # # capabilities: # add: ["SYS_PTRACE"] # # To use SYS_PTRACE capability and collect IO metrics for all processes # you need to configure apparmour. See issue: # https://github.com/moby/moby/issues/21051 (how to make it work in Docker) # Which requires `--security-opt apparmor:unconfined` # But following kubernetes security context documentation it is not possible # To set without manually creating apparmor profile # https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ # https://kubernetes.io/docs/tutorials/clusters/apparmor/ # `unconfined` will be available in future Kubernetes versions # https://github.com/kubernetes/kubernetes/pull/52395 privileged: true # Define your resources if you need. Defaults should be fine for most. # You can lower or increase based on your hosts. resources: limits: cpu: 2 memory: 512Mi requests: cpu: 100m memory: 128Mi # This way we set Kubernetes Node name to be able to filter out # Only pods from current pod. env: - name: KUBERNETES_NODENAME valueFrom: fieldRef: fieldPath: spec.nodeName volumeMounts: # We store state in /data folder (file positions) - name: collectorforkubernetes-state mountPath: /data # Configuration file deployed with ConfigMap - name: collectorforkubernetes-config mountPath: /config/ readOnly: true # Cgroup filesystem to get metrics - name: cgroup mountPath: /rootfs/sys/fs/cgroup readOnly: true # Proc filesystem to get metrics - name: proc mountPath: /rootfs/proc readOnly: true # Docker logs (JSON files) - name: docker-logs mountPath: /rootfs/var/lib/docker/containers/ readOnly: true # Docker socket - name: docker-unix-socket mountPath: /rootfs/var/run/docker.sock readOnly: true # Host logs - name: logs mountPath: /rootfs/var/log/ readOnly: true volumes: # We store state directly on host, change this location, if # your persistent volume is somewhere else - name: collectorforkubernetes-state hostPath: path: /var/lib/collectorforkubernetes/data/ # Location of docker logs - name: docker-logs hostPath: path: /var/lib/docker/containers/ # Location of cgroups file system - name: cgroup hostPath: path: /sys/fs/cgroup # Location of proc file system - name: proc hostPath: path: /proc # Host logs location - name: logs hostPath: path: /var/log # Docker socket - name: docker-unix-socket hostPath: path: /var/run/docker.sock # configuration from ConfigMap - name: collectorforkubernetes-config configMap: name: collectorforkubernetes