MLE-15979 add missing reconciliation logic in enhance deployment (#14)

* MLE-15979: add missing reconciliation logic for MarkLogic Group

* add hugepage resource to unit test

* add complete sample

* update to CRD

* fix testing as per review comment

* Implementation for huge page support

* Add testing for Hugepages

* remove debug logging

---------

Co-authored-by: Peng Zhou <[email protected]>

Author: pengzhouml

api/v1alpha1/common_types.go

@@ -19,12 +19,17 @@ type ContainerProbe struct {
 }
 
 // Storage is the inteface to add pvc and pv support in marklogic
-
 type Storage struct {
 	Size        string             `json:"size,omitempty"`
 	VolumeMount VolumeMountWrapper `json:"volumeMount,omitempty"`
 }
 
+type HugePages struct {
+	Enabled bool `json:"enabled,omitempty"`
+	// +kubebuilder:default:="/dev/hugepages"
+	MountPath string `json:"mountPath,omitempty"`
+}
+
 type VolumeMountWrapper struct {
 	Volume    []corev1.Volume      `json:"volume,omitempty"`
 	MountPath []corev1.VolumeMount `json:"mountPath,omitempty"`

api/v1alpha1/marklogicgroup_types.go

@@ -57,6 +57,9 @@ type MarklogicGroupSpec struct {
 	TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"`
 	PriorityClassName         string                            `json:"priorityClassName,omitempty"`
 
+	// +kubebuilder:default:={enabled: false, mountPath: "/dev/hugepages"}
+	HugePages *HugePages `json:"hugePages,omitempty"`
+
 	// +kubebuilder:default:={enabled: true, initialDelaySeconds: 30, timeoutSeconds: 5, periodSeconds: 30, successThreshold: 1, failureThreshold: 3}
 	LivenessProbe ContainerProbe `json:"livenessProbe,omitempty"`
 	// +kubebuilder:default:={enabled: false, initialDelaySeconds: 10, timeoutSeconds: 5, periodSeconds: 30, successThreshold: 1, failureThreshold: 3}

api/v1alpha1/zz_generated.deepcopy.go

@@ -0,0 +1,81 @@
+apiVersion: database.marklogic.com/v1alpha1
+kind: MarklogicGroup
+metadata:
+  labels:
+    app.kubernetes.io/name: marklogicgroup
+    app.kubernetes.io/instance: marklogicgroup-sample
+    app.kubernetes.io/part-of: marklogic-kubernetes-operator
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/created-by: marklogic-kubernetes-operator
+  name: marklogicgroup-sample
+spec:
+  replicas: 1
+  name: marklogic
+  image: "progressofficial/marklogic-db:11.3.0-ubi-rootless"
+  auth:
+    adminUsername: user
+    adminPassword: pass
+  # storage:
+  #   size: 10Gi
+  terminationGracePeriodSeconds: 9
+  updateStrategy: OnDelete
+  clusterDomain: cluster.local
+  priorityClassName: "system-node-critical"
+  # groupConfig:
+  #   name: "node"
+  enableConverters: true
+  hugePages:
+    enabled: true
+    mountPath: /dev/hugepages
+  nodeSelector:
+    node-role.kubernetes.io/master: "true"
+  affinity: 
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 100
+        podAffinityTerm:
+          labelSelector:
+            matchExpressions:
+            - key: app.kubernetes.io/name
+              operator: In
+              values:
+              - marklogic
+          topologyKey: kubernetes.io/hostname
+  topologySpreadConstraints:
+    - maxSkew: 1
+      topologyKey: kubernetes.io/hostname
+      whenUnsatisfiable: DoNotSchedule
+      labelSelector:
+        matchLabels:
+          app.kubernetes.io/name: marklogic
+    - maxSkew: 1
+      topologyKey: topology.kubernetes.io/zone
+      whenUnsatisfiable: ScheduleAnyway
+      labelSelector:
+        matchLabels:
+          app.kubernetes.io/name: marklogic
+  resources:
+    requests:
+      memory: "4Gi"
+      cpu: "1"
+      # When using huge pages requests should equal limits
+      # refer to https://kubernetes.io/docs/tasks/manage-hugepages/scheduling-hugepages/ for more information on huge pages
+      # hugepages-2Mi: 100Mi
+    limits:
+      memory: "4Gi"
+      cpu: "1"
+      # hugepages-2Mi: 100Mi
+  livenessProbe:
+    enabled: true
+    initialDelaySeconds: 31
+    periodSeconds: 11
+    timeoutSeconds: 6
+    successThreshold: 1
+    failureThreshold: 4
+  readinessProbe:
+    enabled: true
+    initialDelaySeconds: 29
+    periodSeconds: 9
+    timeoutSeconds: 4
+    successThreshold: 1
+    failureThreshold: 2

charts/marklogic-operator/templates/marklogiccluster-crd.yaml

@@ -42,17 +42,27 @@ const (
 )
 
 var replicas = int32(2)
-var cpu = int64(1)
-var memory = int64(268435456)
+
+const resourceCpuValue = int64(1)
+const resourceMemoryValue = int64(268435456)
+
+// 100Mi
+const resourceHugepageValue = int64(104857600)
+
 var typeNamespaceName = types.NamespacedName{Name: Name, Namespace: Namespace}
 
-const imageName = "marklogicdb/marklogic-db:11.1.0-centos-1.1.1"
+const imageName = "progressofficial/marklogic-db:11.3.0-ubi-rootless"
 
 var groupConfig = databasev1alpha1.GroupConfig{
 	Name:          "dnode",
 	EnableXdqpSsl: true,
 }
 
+var hugePages = databasev1alpha1.HugePages{
+	Enabled:   true,
+	MountPath: "/dev/hugepages",
+}
+
 var _ = Describe("MarkLogicGroup controller", func() {
 	Context("When creating an MarklogicGroup", func() {
 		ctx := context.Background()
@@ -79,8 +89,9 @@ var _ = Describe("MarkLogicGroup controller", func() {
 					Image:                     imageName,
 					GroupConfig:               groupConfig,
 					EnableConverters:          true,
+					HugePages:                 &hugePages,
 					UpdateStrategy:            "OnDelete",
-					Resources:                 &corev1.ResourceRequirements{Requests: corev1.ResourceList{"cpu": resource.MustParse("100m"), "memory": resource.MustParse("256Mi")}, Limits: corev1.ResourceList{"cpu": resource.MustParse("100m"), "memory": resource.MustParse("256Mi")}},
+					Resources:                 &corev1.ResourceRequirements{Requests: corev1.ResourceList{"cpu": resource.MustParse("100m"), "memory": resource.MustParse("256Mi"), "hugepages-2Mi": resource.MustParse("100Mi")}, Limits: corev1.ResourceList{"cpu": resource.MustParse("100m"), "memory": resource.MustParse("256Mi"), "hugepages-2Mi": resource.MustParse("100Mi")}},
 					PriorityClassName:         "high-priority",
 					ClusterDomain:             "cluster.local",
 					TopologySpreadConstraints: []corev1.TopologySpreadConstraint{{MaxSkew: 2, TopologyKey: "kubernetes.io/hostname", WhenUnsatisfiable: corev1.ScheduleAnyway}},
@@ -100,10 +111,16 @@ var _ = Describe("MarkLogicGroup controller", func() {
 			Expect(createdCR.Name).Should(Equal(Name))
 			Expect(createdCR.Spec.GroupConfig).Should(Equal(groupConfig))
 			Expect(createdCR.Spec.EnableConverters).Should(Equal(true))
-			Expect(createdCR.Spec.Resources.Limits.Cpu().Value()).Should(Equal(cpu))
-			Expect(createdCR.Spec.Resources.Limits.Memory().Value()).Should(Equal(memory))
-			Expect(createdCR.Spec.Resources.Requests.Cpu().Value()).Should(Equal(cpu))
-			Expect(createdCR.Spec.Resources.Requests.Memory().Value()).Should(Equal(memory))
+			Expect(createdCR.Spec.HugePages.Enabled).Should(Equal(true))
+			Expect(createdCR.Spec.HugePages.MountPath).Should(Equal("/dev/hugepages"))
+			Expect(createdCR.Spec.Resources.Limits.Cpu().Value()).Should(Equal(resourceCpuValue))
+			Expect(createdCR.Spec.Resources.Limits.Memory().Value()).Should(Equal(resourceMemoryValue))
+			hugepagesLimit := createdCR.Spec.Resources.Limits["hugepages-2Mi"]
+			Expect(hugepagesLimit.Value()).Should(Equal(resourceHugepageValue))
+			Expect(createdCR.Spec.Resources.Requests.Cpu().Value()).Should(Equal(resourceCpuValue))
+			Expect(createdCR.Spec.Resources.Requests.Memory().Value()).Should(Equal(resourceMemoryValue))
+			hugepagesRequest := createdCR.Spec.Resources.Requests["hugepages-2Mi"]
+			Expect(hugepagesRequest.Value()).Should(Equal(resourceHugepageValue))
 			Expect(createdCR.Spec.UpdateStrategy).Should(Equal(appsv1.OnDeleteStatefulSetStrategyType))
 			Expect(createdCR.Spec.PriorityClassName).Should(Equal("high-priority"))
 			Expect(createdCR.Spec.ClusterDomain).Should(Equal("cluster.local"))

charts/marklogic-operator/templates/marklogicgroup-crd.yaml

@@ -21,10 +21,13 @@ import (
 type statefulSetParameters struct {
 	Replicas                      *int32
 	Name                          string
-	Metadata                      metav1.ObjectMeta
 	PersistentVolumeClaim         corev1.PersistentVolumeClaim
 	TerminationGracePeriodSeconds *int64
 	UpdateStrategy                appsv1.StatefulSetUpdateStrategyType
+	NodeSelector                  map[string]string
+	Affinity                      *corev1.Affinity
+	TopologySpreadConstraints     []corev1.TopologySpreadConstraint
+	PriorityClassName             string
 }
 
 type containerParameters struct {
@@ -43,6 +46,8 @@ type containerParameters struct {
 	LivenessProbe      databasev1alpha1.ContainerProbe
 	ReadinessProbe     databasev1alpha1.ContainerProbe
 	GroupConfig        databasev1alpha1.GroupConfig
+	EnableConverters   bool
+	HugePages          *databasev1alpha1.HugePages
 }
 
 func (oc *OperatorContext) ReconcileStatefulset() (reconcile.Result, error) {
@@ -188,7 +193,11 @@ func generateStatefulSetsDef(stsMeta metav1.ObjectMeta, params statefulSetParame
 				Spec: corev1.PodSpec{
 					Containers:                    generateContainerDef(stsMeta.GetName(), containerParams),
 					TerminationGracePeriodSeconds: params.TerminationGracePeriodSeconds,
-					Volumes:                       generateVolumes(stsMeta.Name),
+					Volumes:                       generateVolumes(stsMeta.Name, containerParams),
+					NodeSelector:                  params.NodeSelector,
+					Affinity:                      params.Affinity,
+					TopologySpreadConstraints:     params.TopologySpreadConstraints,
+					PriorityClassName:             params.PriorityClassName,
 				},
 			},
 		},
@@ -227,17 +236,18 @@ func generateContainerDef(name string, containerParams containerParameters) []co
 			ImagePullPolicy: containerParams.ImagePullPolicy,
 			Env:             getEnvironmentVariables(containerParams),
 			Lifecycle:       getLifeCycle(),
-			VolumeMounts:    getVolumeMount(),
+			VolumeMounts:    getVolumeMount(containerParams),
 		},
 	}
 	if containerParams.Resources != nil {
 		containerDef[0].Resources = *containerParams.Resources
 	}
-	if containerParams.LivenessProbe.Enabled == true {
+
+	if containerParams.LivenessProbe.Enabled {
 		containerDef[0].LivenessProbe = getLivenessProbe(containerParams.LivenessProbe)
 	}
 
-	if containerParams.ReadinessProbe.Enabled == true {
+	if containerParams.ReadinessProbe.Enabled {
 		containerDef[0].ReadinessProbe = getReadinessProbe(containerParams.ReadinessProbe)
 	}
 
@@ -250,6 +260,10 @@ func generateStatefulSetsParams(cr *databasev1alpha1.MarklogicGroup) statefulSet
 		Name:                          cr.Spec.Name,
 		TerminationGracePeriodSeconds: cr.Spec.TerminationGracePeriodSeconds,
 		UpdateStrategy:                cr.Spec.UpdateStrategy,
+		NodeSelector:                  cr.Spec.NodeSelector,
+		Affinity:                      cr.Spec.Affinity,
+		TopologySpreadConstraints:     cr.Spec.TopologySpreadConstraints,
+		PriorityClassName:             cr.Spec.PriorityClassName,
 	}
 	if cr.Spec.Storage != nil {
 		params.PersistentVolumeClaim = generatePVCTemplate(cr.Spec.Storage.Size)
@@ -260,15 +274,16 @@ func generateStatefulSetsParams(cr *databasev1alpha1.MarklogicGroup) statefulSet
 func generateContainerParams(cr *databasev1alpha1.MarklogicGroup) containerParameters {
 	trueProperty := true
 	containerParams := containerParameters{
-		Image:          cr.Spec.Image,
-		Resources:      cr.Spec.Resources,
-		Name:           cr.Spec.Name,
-		Namespace:      cr.Namespace,
-		ClusterDomain:  cr.Spec.ClusterDomain,
-		BootstrapHost:  cr.Spec.BootstrapHost,
-		LivenessProbe:  cr.Spec.LivenessProbe,
-		ReadinessProbe: cr.Spec.ReadinessProbe,
-		GroupConfig:    cr.Spec.GroupConfig,
+		Image:            cr.Spec.Image,
+		Resources:        cr.Spec.Resources,
+		Name:             cr.Spec.Name,
+		Namespace:        cr.Namespace,
+		ClusterDomain:    cr.Spec.ClusterDomain,
+		BootstrapHost:    cr.Spec.BootstrapHost,
+		LivenessProbe:    cr.Spec.LivenessProbe,
+		ReadinessProbe:   cr.Spec.ReadinessProbe,
+		GroupConfig:      cr.Spec.GroupConfig,
+		EnableConverters: cr.Spec.EnableConverters,
 	}
 
 	if cr.Spec.Storage != nil {
@@ -283,6 +298,9 @@ func generateContainerParams(cr *databasev1alpha1.MarklogicGroup) containerParam
 		containerParams.LicenseKey = cr.Spec.License.Key
 		containerParams.Licensee = cr.Spec.License.Licensee
 	}
+	if cr.Spec.HugePages.Enabled {
+		containerParams.HugePages = cr.Spec.HugePages
+	}
 
 	return containerParams
 }
@@ -302,7 +320,7 @@ func getLifeCycle() *corev1.Lifecycle {
 	}
 }
 
-func generateVolumes(stsName string) []corev1.Volume {
+func generateVolumes(stsName string, containerParams containerParameters) []corev1.Volume {
 	volumes := []corev1.Volume{}
 	volumes = append(volumes, corev1.Volume{
 		Name: "helm-scripts",
@@ -322,6 +340,17 @@ func generateVolumes(stsName string) []corev1.Volume {
 			},
 		},
 	})
+	if containerParams.HugePages.Enabled {
+		volumes = append(volumes, corev1.Volume{
+			Name: "huge-pages",
+			VolumeSource: corev1.VolumeSource{
+				EmptyDir: &corev1.EmptyDirVolumeSource{
+					Medium: corev1.StorageMediumHugePages,
+				},
+			},
+		})
+	}
+
 	return volumes
 }
 
@@ -364,6 +393,10 @@ func getEnvironmentVariables(containerParams containerParameters) []corev1.EnvVa
 		Name:  "MARKLOGIC_CLUSTER_TYPE",
 		Value: "bootstrap",
 	},
+		corev1.EnvVar{
+			Name:  "INSTALL_CONVERTERS",
+			Value: strconv.FormatBool(containerParams.EnableConverters),
+		},
 	)
 	if containerParams.LicenseKey != "" {
 		envVars = append(envVars, corev1.EnvVar{
@@ -390,7 +423,7 @@ func getEnvironmentVariables(containerParams containerParameters) []corev1.EnvVa
 	return envVars
 }
 
-func getVolumeMount() []corev1.VolumeMount {
+func getVolumeMount(containerParams containerParameters) []corev1.VolumeMount {
 	var VolumeMounts []corev1.VolumeMount
 
 	// if persistenceEnabled != nil && *persistenceEnabled {
@@ -409,6 +442,14 @@ func getVolumeMount() []corev1.VolumeMount {
 			ReadOnly:  true,
 		},
 	)
+	if containerParams.HugePages.Enabled {
+		VolumeMounts = append(VolumeMounts,
+			corev1.VolumeMount{
+				Name:      "huge-pages",
+				MountPath: containerParams.HugePages.MountPath,
+			},
+		)
+	}
 	return VolumeMounts
 }