aboutsummaryrefslogtreecommitdiff
path: root/logging/fluentd
diff options
context:
space:
mode:
Diffstat (limited to 'logging/fluentd')
-rw-r--r--logging/fluentd/fluent.conf36
-rw-r--r--logging/fluentd/kubernetes.conf201
-rw-r--r--logging/fluentd/systemd.conf53
3 files changed, 290 insertions, 0 deletions
diff --git a/logging/fluentd/fluent.conf b/logging/fluentd/fluent.conf
new file mode 100644
index 0000000..f10d16c
--- /dev/null
+++ b/logging/fluentd/fluent.conf
@@ -0,0 +1,36 @@
+# FIXED
+
+@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf"
+@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf"
+@include kubernetes.conf
+@include conf.d/*.conf
+
+<match **>
+ @type elasticsearch
+ @id out_es
+ @log_level info
+ include_tag_key true
+ host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
+ port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
+ path "#{ENV['FLUENT_ELASTICSEARCH_PATH']}"
+ scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}"
+ ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}"
+ ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION'] || 'TLSv1'}"
+ reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'false'}"
+ reconnect_on_error "#{ENV['FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR'] || 'true'}"
+ reload_on_failure "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_ON_FAILURE'] || 'true'}"
+ log_es_400_reason "#{ENV['FLUENT_ELASTICSEARCH_LOG_ES_400_REASON'] || 'false'}"
+ logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'logstash'}"
+ logstash_format "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_FORMAT'] || 'true'}"
+ index_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_INDEX_NAME'] || 'logstash'}"
+ type_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_TYPE_NAME'] || 'fluentd'}"
+ <buffer>
+ flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}"
+ flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}"
+ chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}"
+ queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}"
+ retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}"
+ retry_forever true
+ </buffer>
+</match>
+
diff --git a/logging/fluentd/kubernetes.conf b/logging/fluentd/kubernetes.conf
new file mode 100644
index 0000000..78465d3
--- /dev/null
+++ b/logging/fluentd/kubernetes.conf
@@ -0,0 +1,201 @@
+# FIXED
+
+<match fluent.**>
+ @type null
+</match>
+
+<source>
+ @type tail
+ @id in_tail_container_logs
+ path /var/log/containers/*.log
+ exclude_path ["/var/log/containers/fluentd*"]
+ pos_file /var/log/fluentd-containers.log.pos
+ tag kubernetes.*
+ read_from_head true
+ <parse>
+ @type multi_format
+ <pattern>
+ format json
+ time_format %Y-%m-%dT%H:%M:%S.%NZ
+ </pattern>
+ <pattern>
+ format regexp
+ time_format %Y-%m-%dT%H:%M:%S.%N%:z
+ expression /^(?<time>.+) (?<stream>stdout|stderr) (?<partial_flag>[FP]) (?<log>.+)$/
+# expression /^(?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z) (?<output>\w+) (?<partial_flag>[FP]) (?<message>.+)$/
+ </pattern>
+ </parse>
+</source>
+
+<source>
+ @type tail
+ @id in_tail_minion
+ path /var/log/salt/minion
+ pos_file /var/log/fluentd-salt.pos
+ tag salt
+ <parse>
+ @type regexp
+ expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
+ time_format %Y-%m-%d %H:%M:%S
+ </parse>
+</source>
+
+<source>
+ @type tail
+ @id in_tail_startupscript
+ path /var/log/startupscript.log
+ pos_file /var/log/fluentd-startupscript.log.pos
+ tag startupscript
+ <parse>
+ @type syslog
+ </parse>
+</source>
+
+<source>
+ @type tail
+ @id in_tail_docker
+ path /var/log/docker.log
+ pos_file /var/log/fluentd-docker.log.pos
+ tag docker
+ <parse>
+ @type regexp
+ expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
+ </parse>
+</source>
+
+<source>
+ @type tail
+ @id in_tail_etcd
+ path /var/log/etcd.log
+ pos_file /var/log/fluentd-etcd.log.pos
+ tag etcd
+ <parse>
+ @type none
+ </parse>
+</source>
+
+<source>
+ @type tail
+ @id in_tail_kubelet
+ multiline_flush_interval 5s
+ path /var/log/kubelet.log
+ pos_file /var/log/fluentd-kubelet.log.pos
+ tag kubelet
+ <parse>
+ @type kubernetes
+ </parse>
+</source>
+
+<source>
+ @type tail
+ @id in_tail_kube_proxy
+ multiline_flush_interval 5s
+ path /var/log/kube-proxy.log
+ pos_file /var/log/fluentd-kube-proxy.log.pos
+ tag kube-proxy
+ <parse>
+ @type kubernetes
+ </parse>
+</source>
+
+<source>
+ @type tail
+ @id in_tail_kube_apiserver
+ multiline_flush_interval 5s
+ path /var/log/kube-apiserver.log
+ pos_file /var/log/fluentd-kube-apiserver.log.pos
+ tag kube-apiserver
+ <parse>
+ @type kubernetes
+ </parse>
+</source>
+
+<source>
+ @type tail
+ @id in_tail_kube_controller_manager
+ multiline_flush_interval 5s
+ path /var/log/kube-controller-manager.log
+ pos_file /var/log/fluentd-kube-controller-manager.log.pos
+ tag kube-controller-manager
+ <parse>
+ @type kubernetes
+ </parse>
+</source>
+
+<source>
+ @type tail
+ @id in_tail_kube_scheduler
+ multiline_flush_interval 5s
+ path /var/log/kube-scheduler.log
+ pos_file /var/log/fluentd-kube-scheduler.log.pos
+ tag kube-scheduler
+ <parse>
+ @type kubernetes
+ </parse>
+</source>
+
+<source>
+ @type tail
+ @id in_tail_rescheduler
+ multiline_flush_interval 5s
+ path /var/log/rescheduler.log
+ pos_file /var/log/fluentd-rescheduler.log.pos
+ tag rescheduler
+ <parse>
+ @type kubernetes
+ </parse>
+</source>
+
+<source>
+ @type tail
+ @id in_tail_glbc
+ multiline_flush_interval 5s
+ path /var/log/glbc.log
+ pos_file /var/log/fluentd-glbc.log.pos
+ tag glbc
+ <parse>
+ @type kubernetes
+ </parse>
+</source>
+
+<source>
+ @type tail
+ @id in_tail_cluster_autoscaler
+ multiline_flush_interval 5s
+ path /var/log/cluster-autoscaler.log
+ pos_file /var/log/fluentd-cluster-autoscaler.log.pos
+ tag cluster-autoscaler
+ <parse>
+ @type kubernetes
+ </parse>
+</source>
+
+# Example:
+# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods"
+# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200"
+<source>
+ @type tail
+ @id in_tail_kube_apiserver_audit
+ multiline_flush_interval 5s
+ path /var/log/kubernetes/kube-apiserver-audit.log
+ pos_file /var/log/kube-apiserver-audit.log.pos
+ tag kube-apiserver-audit
+ <parse>
+ @type multiline
+ format_firstline /^\S+\s+AUDIT:/
+ # Fields must be explicitly captured by name to be parsed into the record.
+ # Fields may not always be present, and order may change, so this just looks
+ # for a list of key="\"quoted\" value" pairs separated by spaces.
+ # Unknown fields are ignored.
+ # Note: We can't separate query/response lines as format1/format2 because
+ # they don't always come one after the other for a given query.
+ format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/
+ time_format %Y-%m-%dT%T.%L%Z
+ </parse>
+</source>
+
+<filter kubernetes.**>
+ @type kubernetes_metadata
+ @id filter_kube_metadata
+</filter>
+
diff --git a/logging/fluentd/systemd.conf b/logging/fluentd/systemd.conf
new file mode 100644
index 0000000..0203734
--- /dev/null
+++ b/logging/fluentd/systemd.conf
@@ -0,0 +1,53 @@
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb
+
+# Logs from systemd-journal for interesting services.
+<source>
+ @type systemd
+ @id in_systemd_kubelet
+ matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
+ <storage>
+ @type local
+ persistent true
+ path /var/log/fluentd-journald-kubelet-cursor.json
+ </storage>
+ <entry>
+ fields_strip_underscores true
+ </entry>
+ read_from_head true
+ tag kubelet
+</source>
+
+# Logs from docker-systemd
+<source>
+ @type systemd
+ @id in_systemd_docker
+ matches [{ "_SYSTEMD_UNIT": "docker.service" }]
+ <storage>
+ @type local
+ persistent true
+ path /var/log/fluentd-journald-docker-cursor.json
+ </storage>
+ <entry>
+ fields_strip_underscores true
+ </entry>
+ read_from_head true
+ tag docker.systemd
+</source>
+
+# Logs from systemd-journal for interesting services.
+<source>
+ @type systemd
+ @id in_systemd_bootkube
+ matches [{ "_SYSTEMD_UNIT": "bootkube.service" }]
+ <storage>
+ @type local
+ persistent true
+ path /var/log/fluentd-journald-bootkube-cursor.json
+ </storage>
+ <entry>
+ fields_strip_underscores true
+ </entry>
+ read_from_head true
+ tag bootkube
+</source>
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-18 15:14:21 +0000