diff options
| -rw-r--r-- | README.md | 9 | ||||
| -rw-r--r-- | logging/README.md | 21 | ||||
| -rw-r--r-- | logging/counter.yaml | 9 | ||||
| -rw-r--r-- | logging/elasticsearch-service.yaml | 16 | ||||
| -rw-r--r-- | logging/elasticsearch-statefulset.yaml | 77 | ||||
| -rw-r--r-- | logging/fluentd.yaml | 98 | ||||
| -rw-r--r-- | logging/fluentd/fluent.conf | 36 | ||||
| -rw-r--r-- | logging/fluentd/kubernetes.conf | 201 | ||||
| -rw-r--r-- | logging/fluentd/systemd.conf | 53 | ||||
| -rw-r--r-- | logging/kibana.yaml | 43 | ||||
| -rw-r--r-- | logging/kube-logging.yaml | 4 | ||||
| -rw-r--r-- | logging/local-path-storage.yaml | 107 |
12 files changed, 674 insertions, 0 deletions
@@ -24,3 +24,12 @@ Getting bearer token kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') +# Local storage + +* https://github.com/rancher/local-path-provisioner + + kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/master/deploy/local-path-storage.yaml + +# Logging + + * https://www.digitalocean.com/community/tutorials/how-to-set-up-an-elasticsearch-fluentd-and-kibana-efk-logging-stack-on-kubernetes diff --git a/logging/README.md b/logging/README.md new file mode 100644 index 0000000..4cd2e6c --- /dev/null +++ b/logging/README.md @@ -0,0 +1,21 @@ +First, install *local-path-provisioner*. + +Then: + + kubectl create -f kube-logging.yaml + kubectl create -f elasticsearch-service.yaml + kubectl create -f elasticsearch-statefulset.yaml + kubectl create -f kibana.yaml + kubectl create -f fluentd.yaml + +To get some logging output: + + kubectl create -f counter.yaml + +To fix broken config: + + kubectl -n kube-system create configmap fluentd-config \ + --from-file fluentd/kubernetes.conf \ + --from-file fluentd/fluent.conf \ + --from-file fluentd/systemd.conf + diff --git a/logging/counter.yaml b/logging/counter.yaml new file mode 100644 index 0000000..e0d1085 --- /dev/null +++ b/logging/counter.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Pod +metadata: + name: counter +spec: + containers: + - name: count + image: busybox + args: [/bin/sh, -c, 'i=0; while true; do echo "$i: $(date)"; i=$((i+1)); sleep 1; done'] diff --git a/logging/elasticsearch-service.yaml b/logging/elasticsearch-service.yaml new file mode 100644 index 0000000..b4c8bbd --- /dev/null +++ b/logging/elasticsearch-service.yaml @@ -0,0 +1,16 @@ +kind: Service +apiVersion: v1 +metadata: + name: elasticsearch + namespace: kube-logging + labels: + app: elasticsearch +spec: + selector: + app: elasticsearch + clusterIP: None + ports: + - port: 9200 + name: rest + - port: 9300 + name: inter-node diff --git a/logging/elasticsearch-statefulset.yaml b/logging/elasticsearch-statefulset.yaml new file mode 100644 index 0000000..543b06a --- /dev/null +++ b/logging/elasticsearch-statefulset.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: es-cluster + namespace: kube-logging +spec: + serviceName: elasticsearch + replicas: 3 + selector: + matchLabels: + app: elasticsearch + template: + metadata: + labels: + app: elasticsearch + spec: + containers: + - name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0 + resources: + limits: + cpu: 1000m + requests: + cpu: 100m + ports: + - containerPort: 9200 + name: rest + protocol: TCP + - containerPort: 9300 + name: inter-node + protocol: TCP + volumeMounts: + - name: data + mountPath: /usr/share/elasticsearch/data + env: + - name: cluster.name + value: k8s-logs + - name: node.name + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: discovery.seed_hosts + value: "es-cluster-0.elasticsearch,es-cluster-1.elasticsearch,es-cluster-2.elasticsearch" + - name: cluster.initial_master_nodes + value: "es-cluster-0,es-cluster-1,es-cluster-2" + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" + initContainers: + - name: fix-permissions + image: busybox + command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"] + securityContext: + privileged: true + volumeMounts: + - name: data + mountPath: /usr/share/elasticsearch/data + - name: increase-vm-max-map + image: busybox + command: ["sysctl", "-w", "vm.max_map_count=262144"] + securityContext: + privileged: true + - name: increase-fd-ulimit + image: busybox + command: ["sh", "-c", "ulimit -n 65536"] + securityContext: + privileged: true + volumeClaimTemplates: + - metadata: + name: data + labels: + app: elasticsearch + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: local-path + resources: + requests: + storage: 10Gi diff --git a/logging/fluentd.yaml b/logging/fluentd.yaml new file mode 100644 index 0000000..089c184 --- /dev/null +++ b/logging/fluentd.yaml @@ -0,0 +1,98 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluentd + namespace: kube-logging + labels: + app: fluentd +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fluentd + labels: + app: fluentd +rules: +- apiGroups: + - "" + resources: + - pods + - namespaces + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: fluentd +roleRef: + kind: ClusterRole + name: fluentd + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: fluentd + namespace: kube-logging +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluentd + namespace: kube-logging + labels: + app: fluentd +spec: + selector: + matchLabels: + app: fluentd + template: + metadata: + labels: + app: fluentd + spec: + serviceAccount: fluentd + serviceAccountName: fluentd + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: fluentd + image: fluent/fluentd-kubernetes-daemonset:v1.4.2-debian-elasticsearch-1.1 + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: "elasticsearch.kube-logging.svc.cluster.local" + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" + - name: FLUENT_ELASTICSEARCH_SCHEME + value: "http" + - name: FLUENTD_SYSTEMD_CONF + value: disable + - name: FLUENT_ELASTICSEARCH_SED_DISABLE + value: yeah + resources: + limits: + memory: 512Mi + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - name: fluentd-config + mountPath: /fluentd/etc/ + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + terminationGracePeriodSeconds: 30 + volumes: + - name: fluentd-config + configMap: + name: fluentd-config + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers diff --git a/logging/fluentd/fluent.conf b/logging/fluentd/fluent.conf new file mode 100644 index 0000000..f10d16c --- /dev/null +++ b/logging/fluentd/fluent.conf @@ -0,0 +1,36 @@ +# FIXED + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type elasticsearch + @id out_es + @log_level info + include_tag_key true + host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}" + port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}" + path "#{ENV['FLUENT_ELASTICSEARCH_PATH']}" + scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}" + ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}" + ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION'] || 'TLSv1'}" + reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'false'}" + reconnect_on_error "#{ENV['FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR'] || 'true'}" + reload_on_failure "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_ON_FAILURE'] || 'true'}" + log_es_400_reason "#{ENV['FLUENT_ELASTICSEARCH_LOG_ES_400_REASON'] || 'false'}" + logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'logstash'}" + logstash_format "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_FORMAT'] || 'true'}" + index_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_INDEX_NAME'] || 'logstash'}" + type_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_TYPE_NAME'] || 'fluentd'}" + <buffer> + flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}" + flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}" + chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}" + queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}" + retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}" + retry_forever true + </buffer> +</match> + diff --git a/logging/fluentd/kubernetes.conf b/logging/fluentd/kubernetes.conf new file mode 100644 index 0000000..78465d3 --- /dev/null +++ b/logging/fluentd/kubernetes.conf @@ -0,0 +1,201 @@ +# FIXED + +<match fluent.**> + @type null +</match> + +<source> + @type tail + @id in_tail_container_logs + path /var/log/containers/*.log + exclude_path ["/var/log/containers/fluentd*"] + pos_file /var/log/fluentd-containers.log.pos + tag kubernetes.* + read_from_head true + <parse> + @type multi_format + <pattern> + format json + time_format %Y-%m-%dT%H:%M:%S.%NZ + </pattern> + <pattern> + format regexp + time_format %Y-%m-%dT%H:%M:%S.%N%:z + expression /^(?<time>.+) (?<stream>stdout|stderr) (?<partial_flag>[FP]) (?<log>.+)$/ +# expression /^(?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z) (?<output>\w+) (?<partial_flag>[FP]) (?<message>.+)$/ + </pattern> + </parse> +</source> + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file /var/log/fluentd-salt.pos + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> + +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file /var/log/fluentd-startupscript.log.pos + tag startupscript + <parse> + @type syslog + </parse> +</source> + +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file /var/log/fluentd-docker.log.pos + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> + +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file /var/log/fluentd-etcd.log.pos + tag etcd + <parse> + @type none + </parse> +</source> + +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file /var/log/fluentd-kubelet.log.pos + tag kubelet + <parse> + @type kubernetes + </parse> +</source> + +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file /var/log/fluentd-kube-proxy.log.pos + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> + +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file /var/log/fluentd-kube-apiserver.log.pos + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> + +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file /var/log/fluentd-kube-controller-manager.log.pos + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> + +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file /var/log/fluentd-kube-scheduler.log.pos + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> + +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file /var/log/fluentd-rescheduler.log.pos + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> + +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file /var/log/fluentd-glbc.log.pos + tag glbc + <parse> + @type kubernetes + </parse> +</source> + +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file /var/log/fluentd-cluster-autoscaler.log.pos + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> + +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file /var/log/kube-apiserver-audit.log.pos + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata +</filter> + diff --git a/logging/fluentd/systemd.conf b/logging/fluentd/systemd.conf new file mode 100644 index 0000000..0203734 --- /dev/null +++ b/logging/fluentd/systemd.conf @@ -0,0 +1,53 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + <entry> + fields_strip_underscores true + </entry> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + <entry> + fields_strip_underscores true + </entry> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + <entry> + fields_strip_underscores true + </entry> + read_from_head true + tag bootkube +</source> diff --git a/logging/kibana.yaml b/logging/kibana.yaml new file mode 100644 index 0000000..3fe9839 --- /dev/null +++ b/logging/kibana.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: Service +metadata: + name: kibana + namespace: kube-logging + labels: + app: kibana +spec: + ports: + - port: 5601 + selector: + app: kibana +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kibana + namespace: kube-logging + labels: + app: kibana +spec: + replicas: 1 + selector: + matchLabels: + app: kibana + template: + metadata: + labels: + app: kibana + spec: + containers: + - name: kibana + image: docker.elastic.co/kibana/kibana:7.2.0 + resources: + limits: + cpu: 1000m + requests: + cpu: 100m + env: + - name: ELASTICSEARCH_URL + value: http://elasticsearch:9200 + ports: + - containerPort: 5601 diff --git a/logging/kube-logging.yaml b/logging/kube-logging.yaml new file mode 100644 index 0000000..805477b --- /dev/null +++ b/logging/kube-logging.yaml @@ -0,0 +1,4 @@ +kind: Namespace +apiVersion: v1 +metadata: + name: kube-logging diff --git a/logging/local-path-storage.yaml b/logging/local-path-storage.yaml new file mode 100644 index 0000000..7d717bd --- /dev/null +++ b/logging/local-path-storage.yaml @@ -0,0 +1,107 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: local-path-storage +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-path-provisioner-service-account + namespace: local-path-storage +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: local-path-provisioner-role + namespace: local-path-storage +rules: +- apiGroups: [""] + resources: ["nodes", "persistentvolumeclaims"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["endpoints", "persistentvolumes", "pods"] + verbs: ["*"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: local-path-provisioner-bind + namespace: local-path-storage +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: local-path-provisioner-role +subjects: +- kind: ServiceAccount + name: local-path-provisioner-service-account + namespace: local-path-storage +--- +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: local-path-provisioner + namespace: local-path-storage +spec: + replicas: 1 + selector: + matchLabels: + app: local-path-provisioner + template: + metadata: + labels: + app: local-path-provisioner + spec: + serviceAccountName: local-path-provisioner-service-account + containers: + - name: local-path-provisioner + image: rancher/local-path-provisioner:v0.0.9 + imagePullPolicy: IfNotPresent + command: + - local-path-provisioner + - --debug + - start + - --config + - /etc/config/config.json + volumeMounts: + - name: config-volume + mountPath: /etc/config/ + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumes: + - name: config-volume + configMap: + name: local-path-config +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local-path +provisioner: rancher.io/local-path +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: local-path-config + namespace: local-path-storage +data: + config.json: |- + { + "nodePathMap":[ + { + "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES", + "paths":["/opt/local-path-provisioner"] + } + ] + } + |