Merge pull request #28 from ilanRosenbaum/feature/CLD-554

CLD-554: Exporting MarkLogic Logs in K8s

Author: ilanRosenbaum

README.md

@@ -179,7 +179,7 @@ You should see an entry named "my-release" (or the release name you chose) with
 
 ## Configuration Options
 
-This section describes the configuration options you can use with Helm. 
+This section describes the configuration options you can use with Helm.
 
 ### --values
 
@@ -221,6 +221,15 @@ helm install my-release marklogic/marklogic --version=1.0.0-ea1 \
 
 We recommend that you use the `values.yaml` file for configuring your installation.
 
+### Log Collection
+
+To enable log collection for all Marklogic logs set logCollection.enabled to true. Set each option in logCollection.files to true of false depending on if you want to track each type of Marklogic log file.
+
+In order to use the logs that are colleceted you must define an output in the outputs section of the values file. Fluent Bit will parse and output all the log files from each pod to the output(s) you set.
+
+For documentation on how to configure the Fluent Bit output with your logging backend see Fluent Bit's output documentation here: <https://docs.fluentbit.io/manual/pipeline/outputs>
+
+
 ## Adding and Removing Hosts from Clusters
 
 ### Adding Hosts
@@ -263,7 +272,6 @@ kubectl logs pod/terminated-host-pod-name
 ```
 
 If you are permanently removing the host from the MarkLogic cluster, once the pod is terminated, follow standard MarkLogic administrative procedures using the administrative UI or APIs to remove the MarkLogic host from the cluster. Also, because Kubernetes keeps the Persistent Volume Claims and Persistent Volumes around until they are explicitly deleted, you must manually delete them using the Kubernetes APIs before attempting to scale the hosts in the StatefulSet back up again.
-
 # Access the MarkLogic Server
 
 ## Service
@@ -412,3 +420,9 @@ This table describes the list of available parameters for Helm Chart.
 | `startupProbe.timeoutSeconds`        | Timeout seconds for startup probe                                                                              | `1`                                  |
 | `startupProbe.failureThreshold`      | Failure threshold for startup probe                                                                            | `30`                                 |
 | `startupProbe.successThreshold`      | Success threshold for startup probe                                                                            | `1`                                  |
+| `logCollection.enabled`              | Enable this parameter to enable cluster wide log collection of Marklogic server logs                           | `false`                              |
+| `logCollection.files.errorLogs`      | Enable this parameter to enable collection of Marklogics error logs when clog collection is enabled            | `true`                               |
+| `logCollection.files.accessLogs`     | Enable this parameter to enable collection of Marklogics access logs when log collection is enabled            | `true`                               |
+| `logCollection.files.requestLogs`    | Enable this parameter to enable collection of Marklogics request logs when log collection is enabled           | `true`                               |
+| `logCollection.files.crashLogs`      | Enable this parameter to enable collection of Marklogics crash logs when log collection is enabled             | `true`                               |
+| `logCollection.files.auditLogs`      | Enable this parameter to enable collection of Marklogics audit logs when log collection is enabled             | `true`                               |

charts/templates/configmap.yaml

@@ -10,3 +10,132 @@ data:
   MARKLOGIC_FQDN_SUFFIX: {{ include "marklogic.headlessURL" . }}
   MARKLOGIC_INIT: "true"
   MARKLOGIC_JOIN_CLUSTER: "true"
+---
+{{- if .Values.logCollection.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "marklogic.fullname" . }}-fb-config-map
+  namespace: {{ .Release.Namespace }}
+data:
+  fluent-bit.conf: |
+      [SERVICE]
+        Flush 5
+        Log_Level info
+        Daemon off
+        Parsers_File parsers.conf
+
+      @INCLUDE inputs.conf
+      @INCLUDE filters.conf
+      @INCLUDE outputs.conf
+
+  inputs.conf: |
+      {{- if .Values.logCollection.files.errorLogs }}
+      [INPUT]
+        Name tail
+        Path {{ .Values.persistence.mountPath }}/Logs/*ErrorLog.txt
+        Read_from_head true
+        Tag kube.marklogic.logs.error
+        Path_Key path
+        Parser error_parser
+        Mem_Buf_Limit 4MB
+      {{- end }}
+      
+      {{- if .Values.logCollection.files.accessLogs }}
+      [INPUT]
+        Name tail
+        Path {{ .Values.persistence.mountPath }}/Logs/*AccessLog.txt
+        Read_from_head true
+        tag kube.marklogic.logs.access
+        Path_Key path
+        Parser access_parser
+        Mem_Buf_Limit 4MB
+      {{- end }}
+
+      {{- if .Values.logCollection.files.requestLogs }}
+      [INPUT]
+        Name tail
+        Path {{ .Values.persistence.mountPath }}/Logs/*RequestLog.txt
+        Read_from_head true
+        tag kube.marklogic.logs.request
+        Path_Key path
+        Parser json_parser
+        Mem_Buf_Limit 4MB
+      {{- end }}
+
+      {{- if .Values.logCollection.files.crashLogs }}
+      [INPUT]
+        Name tail
+        Path {{ .Values.persistence.mountPath }}/Logs/CrashLog.txt
+        Read_from_head true
+        tag kube.marklogic.logs.crash
+        Path_Key path
+        Mem_Buf_Limit 4MB
+      {{- end }}
+
+      {{- if .Values.logCollection.files.auditLogs }}
+      [INPUT]
+        Name tail
+        Path {{ .Values.persistence.mountPath }}/Logs/AuditLog.txt
+        Read_from_head true
+        tag kube.marklogic.logs.audit
+        Path_Key path
+        Mem_Buf_Limit 4MB
+      {{- end }}
+
+  outputs.conf: {{ toYaml .Values.logCollection.outputs | indent 2 }}
+
+  filters.conf: |
+      # Enrich Logs
+      [FILTER]
+        Name modify
+        Match *
+        Add pod ${POD_NAME}
+        Add namespace {{ .Release.Namespace }}
+
+      [FILTER]
+        Name modify
+        Match kube.marklogic.logs.error
+        Add tag kube.marklogic.logs.error
+
+      [FILTER]
+        Name modify
+        Match kube.marklogic.logs.access
+        Add tag kube.marklogic.logs.access
+
+      [FILTER]
+        Name modify
+        Match kube.marklogic.logs.request
+        Add tag kube.marklogic.logs.request
+
+      [FILTER]
+        Name modify
+        Match kube.marklogic.logs.audit
+        Add tag kube.marklogic.logs.audit
+
+      [FILTER]
+        Name modify
+        Match kube.marklogic.logs.crash
+        Add tag kube.marklogic.logs.crash
+
+  parsers.conf: |
+      [PARSER]
+        Name error_parser
+        Format regex
+        Regex ^(?<time>(.+?)(?=[a-zA-Z]))(?<log_level>(.+?)(?=:))(.+?)(?=[a-zA-Z])(?<log>.*)
+        Time_Key time 
+        Time_Format %Y-%m-%d %H:%M:%S.%L
+
+      [PARSER]
+        Name access_parser
+        Format regex
+        Regex ^(?<host>[^ ]*)(.+?)(?<=\- )(?<user>(.+?)(?=\[))(.+?)(?<=\[)(?<time>(.+?)(?=\]))(.+?)(?<=")(?<request>[^\ ]+[^\"]+)(.+?)(?=\d)(?<response_code>[^\ ]*)(.+?)(?=\d|-)(?<response_obj_size>[^\ ]*)(.+?)(?=")(?<request_info>.*)
+        Time_Key time 
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+      [PARSER]
+        Name json_parser
+        Format json
+        Time_Key time
+        Time_Format %Y-%m-%dT%H:%M:%S%z
+{{- end }}

charts/templates/statefulset.yaml

@@ -144,6 +144,25 @@ spec:
           {{- with .Values.resources }}
           resources: {{- toYaml . | nindent 12 }}
           {{- end }}
+        {{- if .Values.logCollection.enabled }}
+        - name: fluent-bit
+          image: {{ .Values.logCollection.image }}
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: datadir
+              mountPath: {{ .Values.persistence.mountPath }}
+            {{- if .Values.extraVolumeMounts }}
+              {{- toYaml .Values.extraVolumeMounts | nindent 12 }}
+            {{- end }}
+            - name: {{ include "marklogic.fullname" . }}-fb-config-map
+              mountPath: /fluent-bit/etc/
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          resources: {{- toYaml .Values.logCollection.resources | nindent 12 }}
+        {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector: {{- toYaml . | nindent 8}}
       {{- end }}
@@ -154,8 +173,13 @@ spec:
       imagePullSecrets:
         - name: {{ include "marklogic.fullname" . }}-registry
       {{- end }}
-      {{- if or (not .Values.persistence.enabled) (.Values.extraVolumes) }}
       volumes:
+        {{- if .Values.logCollection.enabled }}
+        - name: {{ include "marklogic.fullname" . }}-fb-config-map
+          configMap:
+            name: {{ include "marklogic.fullname" . }}-fb-config-map
+        {{- end }}
+      {{- if or (not .Values.persistence.enabled) (.Values.extraVolumes) }}
         {{- if not .Values.persistence.enabled }}
         - name: datadir
           emptyDir: {}

charts/values.yaml

@@ -117,3 +117,33 @@ startupProbe:
   failureThreshold: 30
   successThreshold: 1
 
+# Configure options for log collection
+# Log collection will collect all logs for each file type enabled, parse them, 
+# And export them to a logging backend specified in the outputs section below
+logCollection:
+  enabled: true
+  image: fluent/fluent-bit:1.9.7
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
+    limits:
+      cpu: "100m"
+      memory: "128Mi"
+  files:
+    errorLogs: true
+    accessLogs: true
+    requestLogs: true
+    crashLogs: true
+    auditLogs: true
+  outputs: |-
+      # Example output config for ElasticSearch output
+      # [OUTPUT]
+      #   Name es
+      #   Match *
+      #   Host elasticsearch.default.svc.cluster.local
+      #   Port 9200
+      #   HTTP_User admin
+      #   HTTP_Passwd admin
+      # Documentation on how to set up output can be found here: https://docs.fluentbit.io/manual/pipeline/outputs
+      # Configure desired output below

test/e2e/clustering_test.go

@@ -15,7 +15,6 @@ import (
 	digest_auth "github.com/xinsnake/go-http-digest-auth-client"
 )
 
-
 func TestClusterJoin(t *testing.T) {
 	// Path to the helm chart we will test 
 	helmChartPath, e := filepath.Abs("../../charts")
@@ -38,6 +37,7 @@ func TestClusterJoin(t *testing.T) {
 			"image.tag":           "10-internal",
 			"auth.adminUsername":  username,
 			"auth.adminPassword":  password,
+			"logCollection.enabled":    "false",
 		},
 	}
 

test/e2e/install_test.go

@@ -29,6 +29,7 @@ func TestHelmInstall(t *testing.T) {
 			"replicaCount":        "1",
 			"image.repository":    "marklogic-centos/marklogic-server-centos",
 			"image.tag":           "10-internal",
+			"logCollection.enabled":    "false",
 		},
 	}
 

test/e2e/upgrade_test.go

@@ -29,6 +29,7 @@ func TestHelmUpgrade(t *testing.T) {
 			"replicaCount":        "1",
 			"image.repository":    "marklogic-centos/marklogic-server-centos",
 			"image.tag":           "10-internal",
+			"logCollection.enabled":    "false",
 		},
 	}
 
@@ -48,6 +49,7 @@ func TestHelmUpgrade(t *testing.T) {
 			"replicaCount":        "2",
 			"image.repository":    "marklogic-centos/marklogic-server-centos",
 			"image.tag":           "10-internal",
+			"logCollection.enabled":    "false",
 		},
 	}
 

test/template/template_test.go

@@ -13,7 +13,7 @@ import (
 	"github.com/gruntwork-io/terratest/modules/random"
 )
 
-func TestChartTemplate(t *testing.T) {
+func TestChartTemplateNoLogCollection(t *testing.T) {
 	t.Parallel()
 
 	// Path to the helm chart we will test
@@ -29,9 +29,10 @@ func TestChartTemplate(t *testing.T) {
 	// Setup the args for helm install
 	options := &helm.Options{
 		SetValues: map[string]string{
-			"image.repository":    "store/marklogicdb/marklogic-server",
-			"image.tag":           "tag: 10.0-8.3-centos-1.0.0-ea3-test",
+			"image.repository":    "marklogicdb/marklogic-db",
+			"image.tag":           "latest",
 			"persistence.enabled": "false",
+			"logCollection.enabled":    "false",
 		},
 		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
 	}
@@ -47,8 +48,52 @@ func TestChartTemplate(t *testing.T) {
 	require.Equal(t, namespaceName, statefulset.Namespace)
 
 	// Verify the image matches
-	expectedImage := "store/marklogicdb/marklogic-server:tag: 10.0-8.3-centos-1.0.0-ea3-test"
+	expectedImage := "marklogicdb/marklogic-db:latest"
 	statefulSetContainers := statefulset.Spec.Template.Spec.Containers
 	require.Equal(t, len(statefulSetContainers), 1)
 	require.Equal(t, statefulSetContainers[0].Image, expectedImage)
 }
+
+func TestChartTemplateLogCollection(t *testing.T) {
+	t.Parallel()
+
+	// Path to the helm chart we will test
+	helmChartPath, err := filepath.Abs("../../charts")
+	releaseName := "marklogic-test"
+	t.Log(helmChartPath, releaseName)
+	require.NoError(t, err)
+
+	// Set up the namespace; confirm that the template renders the expected value for the namespace.
+	namespaceName := "marklogic-" + strings.ToLower(random.UniqueId())
+	t.Logf("Namespace: %s\n", namespaceName)
+
+	// Setup the args for helm install
+	options := &helm.Options{
+		SetValues: map[string]string{
+			"image.repository":    "marklogicdb/marklogic-db",
+			"image.tag":           "latest",
+			"persistence.enabled": "false",
+			"logCollection.enabled":    "true",
+		},
+		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+	}
+
+	// render the tempate
+	output := helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/statefulset.yaml"})
+
+	var statefulset appsv1.Deployment
+	helm.UnmarshalK8SYaml(t, output, &statefulset)
+	// t.Log(statefulset)
+
+	// Verify the name and namespace matches
+	require.Equal(t, namespaceName, statefulset.Namespace)
+
+	// Verify the image matches
+	expectedImage1 := "marklogicdb/marklogic-db:latest"
+	expectedImage2 := "fluent/fluent-bit:1.9.7"
+
+	statefulSetContainers := statefulset.Spec.Template.Spec.Containers
+	require.Equal(t, len(statefulSetContainers), 2)
+	require.Equal(t, statefulSetContainers[0].Image, expectedImage1)
+	require.Equal(t, statefulSetContainers[1].Image, expectedImage2)
+}