diff --git a/docs/CustomResources.md b/docs/CustomResources.md index f2587205c..450a4386c 100644 --- a/docs/CustomResources.md +++ b/docs/CustomResources.md @@ -403,6 +403,65 @@ spec: replicas: 1 ``` +#### admin-managed-pv Annotations +The admin-managed-pv annotation in the splunk-operator's Custom Resource allows the admin to control whether Persistent Volumes (PVs) are dynamically created for the StatefulSet associated with the CR. If set to `true`, no PVs will be created, and the Persistent Volume Claim templates in the StatefulSet manifest will include a selector block to match `app.kubernetes.io/instance` and `app.kubernetes.io/name` labels for pre-created PVs. This means that `/opt/splunk/etc` and `/opt/splunk/var` related PVCs will contain code block like below + +``` +apiVersion: v1 +kind: PersistentVolumeClaim +... + selector: + matchLabels: + app.kubernetes.io/instance: splunk-cm-cluster-manager + app.kubernetes.io/name: cluster-manager +``` + +To match selector definition like this, Persistent Volume must set labels accordingly + +``` +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-example-etc + labels: + app.kubernetes.io/instance: splunk-cm-cluster-manager + app.kubernetes.io/name: cluster-manager +``` + +When admin-managed-pv is set to `false`, PVs will be dynamically created as usual, providing dedicated persistent storage for the StatefulSet. + +Here is an example of a Standalone with the admin-managed-pv annotation set. After +``` +apiVersion: enterprise.splunk.com/v4 +kind: Standalone +metadata: + name: single + finalizers: + - enterprise.splunk.com/delete-pvc + annotations: + enterprise.splunk.com/admin-managed-pv: "true" +``` +##### PV label values +In order to prepare labels for CR's persistent volumes you need to know values beforehand +Below is a table listing `app.kubernetes.io/name` values mapped to CRDs +| Customer Resource Definition | app.kubernetes.io/name value | +| ----------- | --------- | +| clustermanager.enterprise.splunk.com | cluster-manager | +| clustermaster.enterprise.splunk.com | cluster-master | +| indexercluster.enterprise.splunk.com | indexer-cluster | +| licensemanager.enterprise.splunk.com | license-manager | +| licensemaster.enterprise.splunk.com | license-master | +| monitoringconsole.enterprise.splunk.com | monitoring-console | +| searchheadcluster.enterprise.splunk.com | search-head | +| standalone.enterprise.splunk.com | standalone | + +`app.kubernetes.io/instance` value consist of three elements concatenated with hyphens +1. "splunk" +2. provided by admin CR name +3. CRD kind name + +For example `clusterManager` CR named "test" will have set `app.kubernetes.io/instance` as `splunk-test-cluster-manager` + #### Container Logs The Splunk Enterprise CRDs deploy Splunkd in Kubernetes pods running [docker-splunk](https://github.com/splunk/docker-splunk) container images. Adding a couple of environment variables to the CR spec as follows produces `detailed container logs`: diff --git a/pkg/splunk/enterprise/configuration.go b/pkg/splunk/enterprise/configuration.go index 7d0302e1a..e06ea7071 100644 --- a/pkg/splunk/enterprise/configuration.go +++ b/pkg/splunk/enterprise/configuration.go @@ -23,6 +23,7 @@ import ( "path/filepath" "reflect" "strconv" + "strings" orderedmap "github.com/wk8/go-ordered-map/v2" appsv1 "k8s.io/api/apps/v1" @@ -98,52 +99,82 @@ func getSplunkLabels(instanceIdentifier string, instanceType InstanceType, partO } // getSplunkVolumeClaims returns a standard collection of Kubernetes volume claims. -func getSplunkVolumeClaims(cr splcommon.MetaObject, spec *enterpriseApi.CommonSplunkSpec, labels map[string]string, volumeType string) (corev1.PersistentVolumeClaim, error) { +func getSplunkVolumeClaims(cr splcommon.MetaObject, spec *enterpriseApi.CommonSplunkSpec, labels map[string]string, volumeType string, adminManagedPV bool) (corev1.PersistentVolumeClaim, error) { var storageCapacity resource.Quantity var err error - - storageClassName := "" - - // Depending on the volume type, determine storage capacity and storage class name(if configured) - if volumeType == splcommon.EtcVolumeStorage { - storageCapacity, err = splcommon.ParseResourceQuantity(spec.EtcVolumeStorageConfig.StorageCapacity, splcommon.DefaultEtcVolumeStorageCapacity) + var storageClassName string + var volumeClaim corev1.PersistentVolumeClaim + + // Depending on the volume type, determine storage capacity and storage class name (if configured) + switch volumeType { + case splcommon.EtcVolumeStorage: + storageCapacity, err = splcommon.ParseResourceQuantity( + spec.EtcVolumeStorageConfig.StorageCapacity, + splcommon.DefaultEtcVolumeStorageCapacity, + ) if err != nil { return corev1.PersistentVolumeClaim{}, fmt.Errorf("%s: %s", "etcStorage", err) } - if spec.EtcVolumeStorageConfig.StorageClassName != "" { - storageClassName = spec.EtcVolumeStorageConfig.StorageClassName - } - } else if volumeType == splcommon.VarVolumeStorage { - storageCapacity, err = splcommon.ParseResourceQuantity(spec.VarVolumeStorageConfig.StorageCapacity, splcommon.DefaultVarVolumeStorageCapacity) + storageClassName = spec.EtcVolumeStorageConfig.StorageClassName + + case splcommon.VarVolumeStorage: + storageCapacity, err = splcommon.ParseResourceQuantity( + spec.VarVolumeStorageConfig.StorageCapacity, + splcommon.DefaultVarVolumeStorageCapacity, + ) if err != nil { return corev1.PersistentVolumeClaim{}, fmt.Errorf("%s: %s", "varStorage", err) } - if spec.VarVolumeStorageConfig.StorageClassName != "" { - storageClassName = spec.VarVolumeStorageConfig.StorageClassName - } + storageClassName = spec.VarVolumeStorageConfig.StorageClassName } - // Create a persistent volume claim - volumeClaim := corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf(splcommon.PvcNamePrefix, volumeType), - Namespace: cr.GetNamespace(), - Labels: labels, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{"ReadWriteOnce"}, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: storageCapacity, + if adminManagedPV { + volumeClaim.Spec.StorageClassName = nil + + volumeClaim = corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(splcommon.PvcNamePrefix, volumeType), + Namespace: cr.GetNamespace(), + Labels: labels, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{"ReadWriteOnce"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: storageCapacity, + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/name": labels["app.kubernetes.io/name"], + "app.kubernetes.io/instance": labels["app.kubernetes.io/instance"], + }, }, }, - }, - } + } + } else { + volumeClaim = corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(splcommon.PvcNamePrefix, volumeType), + Namespace: cr.GetNamespace(), + Labels: labels, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{"ReadWriteOnce"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: storageCapacity, + }, + }, + }, + } + + if storageClassName != "" { + volumeClaim.Spec.StorageClassName = &storageClassName + } - // Assign storage class name if specified - if storageClassName != "" { - volumeClaim.Spec.StorageClassName = &storageClassName } + return volumeClaim, nil } @@ -483,7 +514,16 @@ func addSplunkVolumeToTemplate(podTemplateSpec *corev1.PodTemplateSpec, name str func addPVCVolumes(cr splcommon.MetaObject, spec *enterpriseApi.CommonSplunkSpec, statefulSet *appsv1.StatefulSet, labels map[string]string, volumeType string) error { // prepare and append persistent volume claims if storage is not ephemeral var err error - volumeClaimTemplate, err := getSplunkVolumeClaims(cr, spec, labels, volumeType) + var adminManagedPV bool + + annotations := cr.GetAnnotations() + + // determine if CR's PVs are managed by an admin + if value, ok := annotations["enterprise.splunk.com/admin-managed-pv"]; ok && strings.ToLower(value) == "true" { + adminManagedPV = true + } + + volumeClaimTemplate, err := getSplunkVolumeClaims(cr, spec, labels, volumeType, adminManagedPV) if err != nil { return err } diff --git a/pkg/splunk/enterprise/configuration_test.go b/pkg/splunk/enterprise/configuration_test.go index 01dbcd260..179a7c1ec 100644 --- a/pkg/splunk/enterprise/configuration_test.go +++ b/pkg/splunk/enterprise/configuration_test.go @@ -74,13 +74,9 @@ func marshalAndCompare(t *testing.T, compare interface{}, method string, want st if err != nil { t.Errorf("%s failed to marshall", err) } - actual := strings.ReplaceAll(string(got), " ", "") want = strings.ReplaceAll(want, " ", "") - if actual != want { - t.Errorf("Method %s, got = %s;\nwant %s", method, got, want) - } - require.JSONEq(t, string(got), want) + require.JSONEq(t, want, string(got)) } func TestGetSplunkService(t *testing.T) { @@ -1431,6 +1427,66 @@ func TestAddStorageVolumes(t *testing.T) { t.Errorf("Unable to idenitfy incorrect VarVolumeStorageConfig resource quantity") } + // test if adminManagedPV logic works + + labels = map[string]string{ + "app.kubernetes.io/component": "indexer", + "app.kubernetes.io/instance": "splunk-CM-cluster-manager", + "app.kubernetes.io/managed-by": "splunk-operator", + "app.kubernetes.io/name": "cluster-manager", + } + + // adjust CR annotations + cr = enterpriseApi.ClusterManager{ + ObjectMeta: metav1.ObjectMeta{ + Name: "CM", + Namespace: "test", + Annotations: map[string]string{ + "enterprise.splunk.com/admin-managed-pv": "true", + }, + Labels: labels, + }, + } + + spec = &enterpriseApi.CommonSplunkSpec{ + EtcVolumeStorageConfig: enterpriseApi.StorageClassSpec{ + StorageCapacity: "35Gi", + StorageClassName: "gp2", + }, + VarVolumeStorageConfig: enterpriseApi.StorageClassSpec{ + StorageCapacity: "25Gi", + StorageClassName: "gp2", + }, + } + + // add labels and annotations to the statefulset configuration + statefulSet = &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-statefulset", + Namespace: cr.GetNamespace(), + Annotations: cr.GetAnnotations(), + Labels: cr.GetLabels(), + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Image: "test", + Name: "splunk", + }, + }, + }, + }, + }, + } + + test(`{"apiVersion":"apps/v1","kind":"StatefulSet","metadata":{"annotations":{"enterprise.splunk.com/admin-managed-pv":"true"},"creationTimestamp":null,"labels":{"app.kubernetes.io/component":"indexer","app.kubernetes.io/instance":"splunk-CM-cluster-manager","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"cluster-manager"},"name":"test-statefulset","namespace":"test"},"spec":{"replicas":1,"selector":null,"serviceName":"","template":{"metadata":{"creationTimestamp":null},"spec":{"containers":[{"image":"test","name":"splunk","resources":{},"volumeMounts":[{"mountPath":"/opt/splunk/etc","name":"pvc-etc"},{"mountPath":"/opt/splunk/var","name":"pvc-var"},{"mountPath":"/mnt/probes","name":"splunk-test-probe-configmap"}]}],"volumes":[{"configMap":{"defaultMode":365,"name":"splunk-test-probe-configmap"},"name":"splunk-test-probe-configmap"}]}},"updateStrategy":{},"volumeClaimTemplates":[{"metadata":{"creationTimestamp":null,"labels":{"app.kubernetes.io/component":"indexer","app.kubernetes.io/instance":"splunk-CM-cluster-manager","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"cluster-manager"},"name":"pvc-etc","namespace":"test"},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"35Gi"}},"selector":{"matchLabels":{"app.kubernetes.io/instance":"splunk-CM-cluster-manager","app.kubernetes.io/name":"cluster-manager"}}},"status":{}},{"metadata":{"creationTimestamp":null,"labels":{"app.kubernetes.io/component":"indexer","app.kubernetes.io/instance":"splunk-CM-cluster-manager","app.kubernetes.io/managed-by":"splunk-operator","app.kubernetes.io/name":"cluster-manager"},"name":"pvc-var","namespace":"test"},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"25Gi"}},"selector":{"matchLabels":{"app.kubernetes.io/instance":"splunk-CM-cluster-manager","app.kubernetes.io/name":"cluster-manager"}}},"status":{}}]},"status":{"availableReplicas":0,"replicas":0}}`) } func TestGetVolumeSourceMountFromConfigMapData(t *testing.T) {