Merge pull request #396 from intelligentfu8/kerberos-ddc

[Feature] support kerberos on ddc

Author: smiletan

api/disaggregated/v1/types.go

@@ -48,6 +48,24 @@ type DorisDisaggregatedClusterSpec struct {
 	// if true, will decommission be node when scale down compute group.
 	// if false, will drop be node when scale down compute group.
 	EnableDecommission bool `json:"enableDecommission,omitempty"`
+
+	// KerberosInfo contains a series of access key files, Provides access to kerberos.
+	KerberosInfo *KerberosInfo `json:"kerberosInfo,omitempty"`
+}
+
+type KerberosInfo struct {
+	// Krb5ConfigMap is the name of configmap within 'krb5.conf'
+	Krb5ConfigMap string `json:"krb5ConfigMap,omitempty"`
+
+	// SecretName is the name of sercet within '*.keytab' files,
+	// refer to the following command to create a Secret :
+	// 	'kubectl create secret generic {secret-name} --from-file=. '
+	KeytabSecretName string `json:"keytabSecretName,omitempty"`
+
+	// KeytabPath is the path where the Secret is finally stored inside the pod. default '/etc/keytab/'.
+	// It is not recommended to modify it unless necessary.
+	// This path is the path filled in when configuring "hadoop.kerberos.keytab".
+	KeytabPath string `json:"keytabPath,omitempty"`
 }
 
 // AdminUser describe administrator for manage components in specified cluster.

pkg/common/utils/resource/pod.go

@@ -562,23 +562,37 @@ func buildKerberosEnv(info *v1.KerberosInfo, config map[string]interface{}, comp
 		return nil
 	}
 
+	return buildKerberosEnvUseSecretMountPath(info.KeytabPath, config, string(componentType))
+}
+
+func BuildKerberosEnvForDDC(info *dv1.KerberosInfo, config map[string]interface{}, componentType dv1.DisaggregatedComponentType)[]corev1.EnvVar {
+	if info == nil {
+		return nil
+	}
+
+	return  buildKerberosEnvUseSecretMountPath(info.KeytabPath, config, string(componentType))
+}
+
+func buildKerberosEnvUseSecretMountPath(keytabPath string, config map[string]interface{}, componentType string) []corev1.EnvVar {
 	var krb5ConfPath string
 	switch componentType {
-	case v1.Component_FE:
+	case string(v1.Component_FE), string(dv1.DisaggregatedFE):
 		krb5ConfPath = kerberos.GetKrb5ConfFromJavaOpts(config)
-	case v1.Component_BE, v1.Component_CN:
+	case string(v1.Component_BE), string(v1.Component_CN), string(dv1.DisaggregatedBE):
 		// be config krb5.conf file must set 'kerberos_krb5_conf_path' in be.conf
 		// https://doris.apache.org/docs/3.0/lakehouse/datalake-analytics/hive?_highlight=kerberos_krb5_conf_path#connect-to-kerberos-enabled-hive
 		if value, exists := config["kerberos_krb5_conf_path"]; exists {
 			krb5ConfPath = value.(string)
 		} else {
 			krb5ConfPath = kerberos.KRB5_DEFAULT_CONFIG
 		}
+	default:
+		klog.Errorf("BuildKerberosEnvUseSecretMountPath, componentType %s not supported.", componentType)
 	}
 
 	keytabFinalUsedPath := keytab_default_mount_path
-	if info.KeytabPath != "" {
-		keytabFinalUsedPath = info.KeytabPath
+	if keytabPath != "" {
+		keytabFinalUsedPath = keytabPath
 	}
 
 	return []corev1.EnvVar{
@@ -900,6 +914,24 @@ func GetMultiSecretVolumeAndVolumeMountWithCommonSpec(cSpec *dv1.CommonSpec) ([]
 }
 
 func getKerberosVolumeAndVolumeMount(kerberosInfo *v1.KerberosInfo) ([]corev1.Volume, []corev1.VolumeMount) {
+	if kerberosInfo == nil {
+		return []corev1.Volume{}, []corev1.VolumeMount{}
+	}
+
+	return getKerberosConfigAndSecretVolumeAndVolumeMount(kerberosInfo.Krb5ConfigMap, kerberosInfo.KeytabSecretName)
+}
+
+//get the kerberos volume and mounts to ddc.
+func GetDv1KerberosVolumeAndVolumeMount(kerberosInfo *dv1.KerberosInfo)([]corev1.Volume, []corev1.VolumeMount) {
+	if kerberosInfo == nil {
+		return []corev1.Volume{}, []corev1.VolumeMount{}
+	}
+
+	return getKerberosConfigAndSecretVolumeAndVolumeMount(kerberosInfo.Krb5ConfigMap, kerberosInfo.KeytabSecretName)
+}
+
+//abstract a base function for dcr and ddc used.
+func getKerberosConfigAndSecretVolumeAndVolumeMount(configMapName, secretName string) ([]corev1.Volume, []corev1.VolumeMount) {
 	var volumes []corev1.Volume
 	var volumeMounts []corev1.VolumeMount
 
@@ -909,7 +941,7 @@ func getKerberosVolumeAndVolumeMount(kerberosInfo *v1.KerberosInfo) ([]corev1.Vo
 		VolumeSource: corev1.VolumeSource{
 			ConfigMap: &corev1.ConfigMapVolumeSource{
 				LocalObjectReference: corev1.LocalObjectReference{
-					Name: kerberosInfo.Krb5ConfigMap,
+					Name: configMapName,
 				},
 			},
 		},
@@ -925,7 +957,7 @@ func getKerberosVolumeAndVolumeMount(kerberosInfo *v1.KerberosInfo) ([]corev1.Vo
 		Name: keytab_volume_name,
 		VolumeSource: corev1.VolumeSource{
 			Secret: &corev1.SecretVolumeSource{
-				SecretName: kerberosInfo.KeytabSecretName,
+				SecretName: secretName,
 			},
 		},
 	})

pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go

@@ -115,6 +115,8 @@ func (dcgs *DisaggregatedComputeGroupsController) NewPodTemplateSpec(ddc *dv1.Do
 		pts.Spec.Volumes = append(pts.Spec.Volumes, secretVolumes...)
 	}
 
+	//add last supplementary spec. if add new config in ddc spec and the config need add in pod, use the follow function to add.
+	dcgs.DisaggregatedSubDefaultController.AddClusterSpecForPodTemplate(dv1.DisaggregatedBE, cvs, &ddc.Spec, &pts)
 	cgUniqueId := selector[dv1.DorisDisaggregatedComputeGroupUniqueId]
 	pts.Spec.Affinity = dcgs.ConstructDefaultAffinity(dv1.DorisDisaggregatedComputeGroupUniqueId, cgUniqueId, pts.Spec.Affinity)
 
@@ -135,7 +137,7 @@ func (dcgs *DisaggregatedComputeGroupsController) NewCGContainer(ddc *dv1.DorisD
 	cmd, args := sub.GetDisaggregatedCommand(dv1.DisaggregatedBE)
 	c.Command = cmd
 	c.Args = args
-	c.Name = "compute"
+	c.Name = sub.BEMainContainerName
 
 	c.Ports = resource.GetDisaggregatedContainerPorts(cvs, dv1.DisaggregatedBE)
 	c.Env = cg.CommonSpec.EnvVars

pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go

@@ -38,6 +38,7 @@ const (
 	//DefaultStorageSize int64 = 107374182400
 	basic_auth_path  = "/etc/basic_auth"
 	auth_volume_name = "basic-auth"
+
 )
 
 func (dfc *DisaggregatedFEController) newFEPodsSelector(ddcName string) map[string]string {
@@ -59,6 +60,8 @@ func (dfc *DisaggregatedFEController) NewStatefulset(ddc *v1.DorisDisaggregatedC
 	spec := ddc.Spec.FeSpec
 	_, _, vcts := dfc.BuildVolumesVolumeMountsAndPVCs(confMap, v1.DisaggregatedFE, &spec.CommonSpec)
 	pts := dfc.NewPodTemplateSpec(ddc, confMap)
+	//add last supplementary spec. if add new config in ddc spec and the config need add in pod, use the follow function to add.
+	dfc.DisaggregatedSubDefaultController.AddClusterSpecForPodTemplate(v1.DisaggregatedFE,confMap, &ddc.Spec, &pts)
 	st := dfc.NewDefaultStatefulset(ddc)
 	//metadata
 	func() {
@@ -124,7 +127,7 @@ func (dfc *DisaggregatedFEController) NewFEContainer(ddc *v1.DorisDisaggregatedC
 	cmd, args := sub.GetDisaggregatedCommand(v1.DisaggregatedFE)
 	c.Command = cmd
 	c.Args = args
-	c.Name = "fe"
+	c.Name = sub.FEMainContainerName
 
 	c.Ports = resource.GetDisaggregatedContainerPorts(cvs, v1.DisaggregatedFE)
 	c.Env = ddc.Spec.FeSpec.CommonSpec.EnvVars

pkg/controller/sub_controller/disaggregated_subcontroller.go

@@ -57,6 +57,8 @@ const (
 	FileCachePathKey                     = "file_cache_path"
 	FileCacheSubConfigPathKey            = "path"
 	FileCacheSubConfigTotalSizeKey       = "total_size"
+	FEMainContainerName = "fe"
+	BEMainContainerName = "compute"
 )
 
 type DisaggregatedSubController interface {
@@ -292,7 +294,46 @@ func (d *DisaggregatedSubDefaultController) GetManagementAdminUserAndPWD(ctx con
 
 }
 
+// add cluster specification on container spec. this is useful to add common spec on different type pods, example: kerberos volume for fe and be.
+func(d *DisaggregatedSubDefaultController) AddClusterSpecForPodTemplate(componentType v1.DisaggregatedComponentType, configMap map[string]interface{}, spec *v1.DorisDisaggregatedClusterSpec, pts *corev1.PodTemplateSpec){
+	var c *corev1.Container
+	switch componentType {
+	case v1.DisaggregatedFE:
+		for	i, _ := range pts.Spec.Containers {
+			if pts.Spec.Containers[i].Name == FEMainContainerName {
+				c = &pts.Spec.Containers[i]
+				break
+			}
+		}
+	case v1.DisaggregatedBE:
+		for i, _ := range pts.Spec.Containers {
+			if pts.Spec.Containers[i].Name == BEMainContainerName {
+				c = &pts.Spec.Containers[i]
+				break
+			}
+		}
 
+	default:
+		klog.Errorf("DisaggregatedSubDefaultController AddClusterSpecForPodTemplate componentType %s not supported.", componentType)
+		return
+	}
+
+	//add pod envs
+	envs := resource.BuildKerberosEnvForDDC(spec.KerberosInfo, configMap, componentType)
+	if len(envs) != 0 {
+		c.Env = append(c.Env, envs...)
+	}
+
+	//add kerberos volumeMounts and volumes
+	volumes, volumeMounts := resource.GetDv1KerberosVolumeAndVolumeMount(spec.KerberosInfo)
+	if len(volumeMounts) != 0 {
+		c.VolumeMounts = append(c.VolumeMounts, volumeMounts...)
+	}
+	if len(volumes) != 0 {
+		pts.Spec.Volumes = append(pts.Spec.Volumes, volumes...)
+	}
+
+}
 
 func (d *DisaggregatedSubDefaultController) BuildVolumesVolumeMountsAndPVCs(confMap map[string]interface{}, componentType v1.DisaggregatedComponentType, commonSpec *v1.CommonSpec) ([]corev1.Volume, []corev1.VolumeMount, []corev1.PersistentVolumeClaim) {
 	if commonSpec.PersistentVolume == nil && len(commonSpec.PersistentVolumes) == 0 {