apiVersion: v1 kind: Project metadata: labels: app: collectorforopenshift name: collectorforopenshift annotations: openshift.io/node-selector: '' openshift.io/description: 'Monitoring OpenShift in Splunk, built by Outcold Solutions' openshift.io/display-name: 'Collector for OpenShift' --- apiVersion: v1 kind: ServiceAccount metadata: labels: app: collectorforopenshift name: collectorforopenshift namespace: collectorforopenshift --- apiVersion: v1 kind: ClusterRole metadata: labels: app: collectorforopenshift name: collectorforopenshift rules: - apiGroups: - "" - apps - batch - extensions - monitoring.coreos.com - apps.openshift.io - build.openshift.io resources: - namespaces - events - cronjobs - daemonsets - deployments - jobs - nodes - pods - replicasets - replicationcontrollers - scheduledjobs - statefulsets - prometheuses - deploymentconfigs - builds - buildconfigs - nodes/metrics verbs: - get - list - watch - nonResourceURLs: - /metrics verbs: - get --- apiVersion: v1 kind: ClusterRoleBinding metadata: labels: app: collectorforopenshift name: collectorforopenshift namespace: collectorforopenshift roleRef: kind: ClusterRole name: collectorforopenshift subjects: - kind: ServiceAccount name: collectorforopenshift namespace: collectorforopenshift --- apiVersion: v1 kind: ConfigMap metadata: name: collectorforopenshift namespace: collectorforopenshift labels: app: collectorforopenshift data: 001-general.conf: | # The general configuration is used for all deployments # # Run collector with the flag `-conf` and specify location of the configuration files. # # You can override all the values using environment variables with the format like # COLLECTOR__=
__= # As an example you can set `dataPath` in the `[general]` section as # COLLECTOR__DATAPATH=general__dataPath=C:\\some\\path\\data.db # This parameter can be configured using -env-override, set it to empty string to disable this feature [general] # Review EULA https://www.outcoldsolutions.com/docs/license-agreement/ # and accept EULA by changing the value to *true* acceptEULA = false # Location for the database # Collector stores positions of the files and internal state dataPath = ./data/ # log level (accepted values are trace, debug, info, warn, error, fatal) logLevel = info # http server gives access to two endpoints # /healthz # /metrics httpServerBinding = # telemetry report endpoint, set it to empty string to disable telemetry telemetryEndpoint = https://license.outcold.solutions/telemetry/ # license check endpoint licenseEndpoint = https://license.outcold.solutions/license/ # license server through proxy licenseServerProxyUrl = # license key license = # Node name is used by default as hostname # use this configuration to override hostname = # Include custom fields to attach to every event, in example below every event sent to Splunk will hav # indexed field my_environment=dev. Fields names should match to ^[a-z][_a-z0-9]*$ # Better way to configure that is to specify labels for OpenShift Nodes. # ; fields.my_environment = dev # connection to kubernetes api [general.kubernetes] # name of openshift node (required, by default is configured with environment variables for Pod) nodeName = tokenPath = /var/run/secrets/kubernetes.io/serviceaccount/token certPath = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt # read timeouts timeout = 5s # In case if pod metadata was not retrievied. how often collector should retry to reload the pod metadata metadataFetchRetry = 5s # In case if event is recent, how long pipeline should wait for the metadata to be available in Kubernetes API metadataFetchWait = 30s # In case if collector does not see new events for specific container and with the last metadata refresh # We have not found this container - fow how long we should keep this metadata in cache. metadataTTL = 5m # Splunk output [output.splunk] # Splunk HTTP Event Collector url (required) url = # Splunk HTTP Event Collector Token (required) token = # Allow invalid SSL server certificate insecure = false # Path to CA cerificate caPath = # CA Name to verify caName = # Events are batched with the maximum size set by batchSize and staying in pipeline for not longer # than set by frequency frequency = 5s batchSize = 768K # Splunk through proxy proxyUrl = # Splunk acknowledgement url (.../services/collector/ack) ackUrl = # Enable index acknowledgment ackEnabled = false # Index acknowledgment timeout ackTimeout = 3m # Timeout specifies a time limit for requests made by collector. # The timeout includes connection time, any # redirects, and reading the response body. timeout = 30s 002-daemonset.conf: | # DaemonSet configuration is used for Nodes and Masters. # Connection to the docker host [general.docker] # url for docker API, only unix socket is supported url = unix:///rootfs/var/run/docker.sock # path to docker root folder (can fallback to use folder structure to read docker metadata) dockerRootFolder = /rootfs/var/lib/docker/ # In case if pod metadata was not retrievied. how often collector should retry to reload the pod metadata metadataFetchRetry = 1s # In case if event is recent, how long pipeline should wait for the metadata to be available in Kubernetes API metadataFetchWait = 5s # In case if collector does not see new events for specific container and with the last metadata refresh # We have not found this container - fow how long we should keep this metadata in cache. metadataTTL = 5m # cgroup input [input.system_stats] # disable system level stats disabled = false # cgroups fs location pathCgroups = /rootfs/sys/fs/cgroup # proc location pathProc = /rootfs/proc # how often to collect cgroup stats statsInterval = 30s # override type type = openshift_stats # specify Splunk index index = # proc input [input.proc_stats] # disable proc level stats disabled = false # proc location pathProc = /rootfs/proc # how often to collect proc stats statsInterval = 30s # override type type = openshift_proc_stats # specify Splunk index index = # Container Log files [input.files] # disable container logs monitoring disabled = false # root location of docker files path = /rootfs/var/lib/docker/containers/ # glob matching pattern for log files glob = */*-json.log* # files are read using polling schema, when reach the EOF how often to check if files got updated pollingInterval = 250ms # how often to look for the new files under logs path walkingInterval = 5s # include verbose fields in events (file offset) verboseFields = false # override type type = openshift_logs # specify Splunk index index = # Host logs. Input syslog(.\d+)? files [input.files::syslog] # disable host level logs disabled = false # root location of docker files path = /rootfs/var/log/ # regex matching pattern match = ^(syslog|messages)(.\d+)?$ # limit search only on one level recursive = false # files are read using polling schema, when reach the EOF how often to check if files got updated pollingInterval = 250ms # how often o look for the new files under logs path walkingInterval = 5s # include verbose fields in events (file offset) verboseFields = false # override type type = openshift_host_logs # specify Splunk index index = # field extraction extraction = ^(?P[A-Za-z]+\s+\d+\s\d+:\d+:\d+)\s(?P[^\s]+)\s(?P[^:\[]+)(\[(?P\d+)\])?: (.+)$ # timestamp field timestampField = timestamp # format for timestamp # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006` timestampFormat = Jan 2 15:04:05 # Adjust date, if month/day aren't set in format timestampSetMonth = false timestampSetDay = false # timestamp location (if not defined by format) timestampLocation = Local # Host logs. Input all *.log(.\d+)? files [input.files::logs] # disable host level logs disabled = false # root location of docker files path = /rootfs/var/log/ # regex matching pattern match = ^[\w\-\.]+\.log(.\d+)?$ # files are read using polling schema, when reach the EOF how often to check if files got updated pollingInterval = 250ms # how often o look for the new files under logs path walkingInterval = 5s # include verbose fields in events (file offset) verboseFields = false # override type type = openshift_host_logs # specify Splunk index index = # field extraction extraction = # timestamp field timestampField = # format for timestamp # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006` timestampFormat = # timestamp location (if not defined by format) timestampLocation = # Pipe to join events (container logs only) [pipe.join] # disable joining event disabled = false # Maximum interval of messages in pipeline maxInterval = 100ms # Maximum time to wait for the messages in pipeline maxWait = 1s # Maximum message size maxSize = 100K # Default pattern to indicate new message (should start not from space) patternRegex = ^[^\s] # Fixing multi line events for OpenShift containers [pipe.join::ose-service-catalog-stderr] ; disabled = false matchRegex.openshift_container_image = ^registry.access.redhat.com/openshift\d+/ose-service-catalog:.*$ matchRegex.docker_stream = stderr patternRegex = ^[IWEF]\d{4}\s\d{2}:\d{2}:\d{2}.\d{6}\s [pipe.join::ose-docker-registry-stderr] ; disabled = false matchRegex.openshift_container_image = ^registry.access.redhat.com/openshift\d+/ose-docker-registry:.*$ matchRegex.docker_stream = stderr patternRegex = ^time="[^"]+"\s+level=[^\s]+\s+msg= [pipe.join::ose-docker-registry-stdout] ; disabled = false matchRegex.openshift_container_image = ^registry.access.redhat.com/openshift\d+/ose-docker-registry:.*$ matchRegex.docker_stream = stdout patternRegex = ^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\s+- [pipe.join::ose-stderr] ; disabled = false matchRegex.openshift_container_image = ^registry.access.redhat.com/openshift\d+/ose:.*$ matchRegex.docker_stream = stderr patternRegex = ^[IWEF]\d{4}\s\d{2}:\d{2}:\d{2}.\d{6}\s [pipe.join::ose-stdout] ; disabled = false matchRegex.openshift_container_image = ^registry.access.redhat.com/openshift\d+/ose:.*$ matchRegex.docker_stream = stdout patternRegex = ^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{9}Z\s+ [pipe.join::ose-haproxy-router-stderr] ; disabled = false matchRegex.openshift_container_image = ^registry.access.redhat.com/openshift\d+/ose-haproxy-router:.*$ matchRegex.docker_stream = stderr patternRegex = ^[IWEF]\d{4}\s\d{2}:\d{2}:\d{2}.\d{6}\s # Define special event join patterns for matched events # Section consist of [pipe.join::] # [pipe.join::my_app] ## Set match pattern for the fields #; matchRegex.docker_container_image = my_app #; matchRegex.docker_stream = stdout ## All events start from '[' #; patternRegex = ^\[\d+ [input.prometheus::kubelet] # disable prometheus kubelet metrics disabled = false # override type type = prometheus # specify Splunk index index = # Override host (environment variables are supported) host = ${KUBERNETES_NODENAME} # Override source source = kubelet # how often to collect prometheus metrics interval = 60s # prometheus endpoint endpoint = https://127.0.0.1:10250/metrics # token for "Authorization: Bearer $(cat tokenPath)" tokenPath = /var/run/secrets/kubernetes.io/serviceaccount/token # server certificate for certificate validation certPath = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt # client certificate for authentication clientCertPath = # Allow invalid SSL server certificate insecure = true # include metrics help with the events # can be useful to explore prometheus metrics includeHelp = false 003-daemonset-master.conf: | [input.prometheus::kubernetes-api] # disable prometheus kubernetes-api input disabled = false # override type type = prometheus # specify Splunk index index = # override host host = ${KUBERNETES_NODENAME} # override source source = kubernetes-api # how often to collect prometheus metrics interval = 60s # prometheus endpoint # at first trying to get it from localhost (that way avoiding load balancer, if multiple) # as fallback using proxy endpoint.1localhost = https://127.0.0.1:8443/metrics endpoint.2kubeapi = https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}/metrics # token for "Authorization: Bearer $(cat tokenPath)" tokenPath = /var/run/secrets/kubernetes.io/serviceaccount/token # server certificate for certificate validation certPath = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt # client certificate for authentication clientCertPath = # Allow invalid SSL server certificate insecure = true # include metrics help with the events includeHelp = false [input.prometheus::controller] # disable prometheus controller metrics disabled = false # override type type = prometheus # specify Splunk index index = # override host host = ${KUBERNETES_NODENAME} # override source source = controller # how often to collect prometheus metrics interval = 60s # prometheus endpoint endpoint.https = https://127.0.0.1:8444/metrics # token for "Authorization: Bearer $(cat tokenPath)" tokenPath = /var/run/secrets/kubernetes.io/serviceaccount/token # server certificate for certificate validation certPath = # client certificate for authentication clientCertPath = clientKeyPath = # Allow invalid SSL server certificate insecure = true # include metrics help with the events includeHelp = false [input.prometheus::etcd] # disable prometheus etcd metrics disabled = false # override type type = prometheus # specify Splunk index index = # override host host = ${KUBERNETES_NODENAME} # override source source = etcd # how often to collect prometheus metricd interval = 60s # prometheus endpoint endpoint.https = https://:2379/metrics # token for "Authorization: Bearer $(cat tokenPath)" tokenPath = # server certificate for certificate validation certPath = /rootfs/etc/origin/master/master.etcd-ca.crt # client certificate for authentication clientCertPath = /rootfs/etc/origin/master/master.etcd-client.crt clientKeyPath = /rootfs/etc/origin/master/master.etcd-client.key # Allow invalid SSL server certificate insecure = true # include metrics help with the events includeHelp = false # Audit logs [input.files::audit-logs] # disable host level logs disabled = false # root location of docker files path = /rootfs/var/lib/origin/openpaas-oscp-audit/ # regex matching pattern match = ^[\w\-\.]+\.log(.\d+)?$ # files are read using polling schema, when reach the EOF how often to check if files got updated pollingInterval = 250ms # how often o look for the new files under logs path walkingInterval = 5s # include verbose fields in events (file offset) verboseFields = false # override type type = openshift_host_logs # specify Splunk index index = # field extraction extraction = # timestamp field timestampField = # format for timestamp # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006` timestampFormat = # timestamp location (if not defined by format) timestampLocation = 004-addon.conf: | [general] # addons can be run in parallel with agents addon = true [input.kubernetes_events] # disable collecting kubernetes events disabled = false # override type type = openshift_events # specify Splunk index index = # Set the timeout for how long request to watch events going to hang reading. eventsWatchTimeout = 30m # Ignore events last seen later that this duration. eventsTTL = 12h --- apiVersion: extensions/v1beta1 kind: DaemonSet metadata: name: collectorforopenshift namespace: collectorforopenshift labels: app: collectorforopenshift spec: # Default updateStrategy is OnDelete. For collector RollingUpdate is suitable # When you update configuration updateStrategy: type: RollingUpdate template: metadata: name: collectorforopenshift labels: daemon: collectorforopenshift spec: hostNetwork: true serviceAccountName: collectorforopenshift # We run this DaemonSet only for Non-Masters affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: node-role.kubernetes.io/master operator: DoesNotExist tolerations: - operator: "Exists" effect: "NoSchedule" - operator: "Exists" effect: "NoExecute" containers: - name: collectorforopenshift # Stick to specific version image: registry.connect.redhat.com/outcoldsolutions/collectorforopenshift:4.0.174.180821 securityContext: privileged: true runAsUser: 0 # Define your resources if you need. Defaults should be fine for most. resources: limits: cpu: 2 memory: 512Mi requests: cpu: 100m memory: 128Mi env: - name: KUBERNETES_NODENAME valueFrom: fieldRef: fieldPath: spec.nodeName volumeMounts: # We store state in /data folder (file positions) - name: collectorforopenshift-state mountPath: /data # Configuration file deployed with ConfigMap - name: collectorforopenshift-config mountPath: /config/ readOnly: true # Cgroup filesystem to get metrics - name: cgroup mountPath: /rootfs/sys/fs/cgroup readOnly: true # Proc filesystem to get metrics - name: proc mountPath: /rootfs/proc readOnly: true # Docker logs (JSON files) - name: docker-logs mountPath: /rootfs/var/lib/docker/containers/ readOnly: true # Docker socket - name: docker-unix-socket mountPath: /rootfs/var/run/docker.sock readOnly: true # Host logs - name: logs mountPath: /rootfs/var/log/ readOnly: true volumes: # We store state directly on host, change this location, if # your persistent volume is somewhere else - name: collectorforopenshift-state hostPath: path: /var/lib/collectorforopenshift/data/ # Location of docker logs - name: docker-logs hostPath: path: /var/lib/docker/containers/ # Location of cgroups file system - name: cgroup hostPath: path: /sys/fs/cgroup # Location of proc file system - name: proc hostPath: path: /proc # Host logs location - name: logs hostPath: path: /var/log # Unix socket - name: docker-unix-socket hostPath: path: /var/run/docker.sock # configuration from ConfigMap - name: collectorforopenshift-config configMap: name: collectorforopenshift items: - key: 001-general.conf path: 001-general.conf - key: 002-daemonset.conf path: 002-daemonset.conf --- apiVersion: extensions/v1beta1 kind: DaemonSet metadata: name: collectorforopenshift-master namespace: collectorforopenshift labels: app: collectorforopenshift spec: updateStrategy: type: RollingUpdate template: metadata: name: collectorforopenshift-master labels: daemon: collectorforopenshift spec: hostNetwork: true serviceAccountName: collectorforopenshift # Deploy only on master affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: node-role.kubernetes.io/master operator: Exists tolerations: - operator: "Exists" effect: "NoSchedule" - operator: "Exists" effect: "NoExecute" containers: - name: collectorforopenshift image: registry.connect.redhat.com/outcoldsolutions/collectorforopenshift:4.0.174.180821 securityContext: privileged: true runAsUser: 0 resources: limits: cpu: 1 memory: 512Mi requests: cpu: 100m memory: 128Mi env: - name: KUBERNETES_NODENAME valueFrom: fieldRef: fieldPath: spec.nodeName volumeMounts: - name: collectorforopenshift-state mountPath: /data - name: collectorforopenshift-config mountPath: /config/ readOnly: true - name: cgroup mountPath: /rootfs/sys/fs/cgroup readOnly: true - name: proc mountPath: /rootfs/proc readOnly: true - name: docker-logs mountPath: /rootfs/var/lib/docker/containers/ readOnly: true - name: docker-unix-socket mountPath: /rootfs/var/run/docker.sock readOnly: true - name: logs mountPath: /rootfs/var/log/ readOnly: true - name: origin-certs mountPath: /rootfs/etc/origin/master/ readOnly: true - name: origin-lib mountPath: /rootfs/var/lib/origin/ readOnly: true volumes: - name: collectorforopenshift-state hostPath: path: /var/lib/collectorforopenshift/data/ - name: docker-logs hostPath: path: /var/lib/docker/containers/ - name: cgroup hostPath: path: /sys/fs/cgroup - name: proc hostPath: path: /proc - name: logs hostPath: path: /var/log - name: docker-unix-socket hostPath: path: /var/run/docker.sock - name: origin-certs hostPath: path: /etc/origin/master/ - name: origin-lib hostPath: path: /var/lib/origin - name: collectorforopenshift-config configMap: name: collectorforopenshift items: - key: 001-general.conf path: 001-general.conf - key: 002-daemonset.conf path: 002-daemonset.conf - key: 003-daemonset-master.conf path: 003-daemonset-master.conf --- apiVersion: apps/v1beta1 kind: Deployment metadata: name: collectorforopenshift-addon namespace: collectorforopenshift labels: app: collectorforopenshift spec: replicas: 1 template: metadata: name: collectorforopenshift-addon labels: app: collectorforopenshift spec: serviceAccountName: collectorforopenshift containers: - name: collectorforopenshift image: registry.connect.redhat.com/outcoldsolutions/collectorforopenshift:4.0.174.180821 securityContext: privileged: true runAsUser: 0 resources: limits: cpu: 500m memory: 256Mi requests: cpu: 50m memory: 32Mi env: - name: KUBERNETES_NODENAME valueFrom: fieldRef: fieldPath: spec.nodeName volumeMounts: - name: collectorforopenshift-state mountPath: /data - name: collectorforopenshift-config mountPath: /config/ readOnly: true volumes: - name: collectorforopenshift-state hostPath: path: /var/lib/collectorforopenshift/data/ - name: collectorforopenshift-config configMap: name: collectorforopenshift items: - key: 001-general.conf path: 001-general.conf - key: 004-addon.conf path: 004-addon.conf