diff --git a/api/disaggregated/v1/types.go b/api/disaggregated/v1/types.go index ef7d1aac..2712ce30 100644 --- a/api/disaggregated/v1/types.go +++ b/api/disaggregated/v1/types.go @@ -121,6 +121,12 @@ type ComputeGroup struct { // Default System Init means that the container must be started in privileged mode. // Default System Init configuration is implemented through the initContainers of the pod, so changes to this configuration may be ignored by k8s when it is not the first deployment. SkipDefaultSystemInit bool `json:"skipDefaultSystemInit,omitempty"` + + // AutoResolveLimitCPU indicates whether to automatically set the CPU limit to doris BE config. + // Default value is 'false'. This means that the Doris BE is unaware of the CPU configuration of resources. + // Enabling this configuration means injecting an ENV named BE_CPU_LIMIT with the value requests.cpu into the pod. This configuration will also appear in the 'be.conf' file inside the BE container. + // Changing this configuration will cause a BE rolling restart. + AutoResolveLimitCPU bool `json:"autoResolveLimitCPU,omitempty"` } type CommonSpec struct { diff --git a/api/doris/v1/types.go b/api/doris/v1/types.go index 4bbb442d..f86d8557 100644 --- a/api/doris/v1/types.go +++ b/api/doris/v1/types.go @@ -23,8 +23,8 @@ import ( ) var ( - AnnotationDebugKey = "selectdb.com.doris/runmode" - AnnotationDebugValue = "debug" + AnnotationDebugKey = "selectdb.com.doris/runmode" + AnnotationDebugValue = "debug" AnnotationDebugDorisKey = "apache.com.doris/runmode" ) @@ -129,6 +129,12 @@ type BeSpec struct { //EnableFeAffinity schedule the be pod on the hosts that have fe pod. when in test situation or have 3 fe and 3 be nodes, and wants one fe and one be in same host. //the weight of antiAffinity in same node is greater than this affinity. EnableFeAffinity bool `json:"enableFeAffinity,omitempty"` + + // AutoResolveLimitCPU indicates whether to automatically set the CPU limit to doris BE config. + // Default value is 'false'. This means that the Doris BE is unaware of the CPU configuration of resources. + // Enabling this configuration means injecting an ENV named BE_CPU_LIMIT with the value requests.cpu into the pod. This configuration will also appear in the 'be.conf' file inside the BE container. + // Changing this configuration will cause a BE rolling restart. + AutoResolveLimitCPU bool `json:"autoResolveLimitCPU,omitempty"` } // FeAddress specify the fe address, please set it when you deploy fe outside k8s or deploy components use crd except fe, if not set . @@ -158,6 +164,12 @@ type CnSpec struct { //AutoScalingPolicy auto scaling strategy AutoScalingPolicy *AutoScalingPolicy `json:"autoScalingPolicy,omitempty"` + + // AutoResolveLimitCPU indicates whether to automatically set the CPU limit to doris BE config. + // Default value is 'false'. This means that the Doris BE is unaware of the CPU configuration of resources. + // Enabling this configuration means injecting an ENV named BE_CPU_LIMIT with the value requests.cpu into the pod. This configuration will also appear in the 'be.conf' file inside the BE container. + // Changing this configuration will cause a BE rolling restart. + AutoResolveLimitCPU bool `json:"autoResolveLimitCPU,omitempty"` } // BrokerSpec describes a template for creating copies of a broker software service, if deploy broker we recommend you add affinity for deploy with be pod. diff --git a/config/crd/bases/crds.yaml b/config/crd/bases/crds.yaml index 1887a9fb..6c776c0a 100644 --- a/config/crd/bases/crds.yaml +++ b/config/crd/bases/crds.yaml @@ -1008,6 +1008,13 @@ spec: description: annotation for fe pods. user can config monitor annotation for collect to monitor system. type: object + autoResolveLimitCPU: + description: |- + AutoResolveLimitCPU indicates whether to automatically set the CPU limit to doris BE config. + Default value is 'false'. This means that the Doris BE is unaware of the CPU configuration of resources. + Enabling this configuration means injecting an ENV named BE_CPU_LIMIT with the value requests.cpu into the pod. This configuration will also appear in the 'be.conf' file inside the BE container. + Changing this configuration will cause a BE rolling restart. + type: boolean claims: description: |- Claims lists the names of resources, defined in spec.resourceClaims, @@ -5093,6 +5100,13 @@ spec: description: annotation for fe pods. user can config monitor annotation for collect to monitor system. type: object + autoResolveLimitCPU: + description: |- + AutoResolveLimitCPU indicates whether to automatically set the CPU limit to doris BE config. + Default value is 'false'. This means that the Doris BE is unaware of the CPU configuration of resources. + Enabling this configuration means injecting an ENV named BE_CPU_LIMIT with the value requests.cpu into the pod. This configuration will also appear in the 'be.conf' file inside the BE container. + Changing this configuration will cause a BE rolling restart. + type: boolean autoScalingPolicy: description: AutoScalingPolicy auto scaling strategy properties: @@ -10146,6 +10160,13 @@ spec: Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. type: object + autoResolveLimitCPU: + description: |- + AutoResolveLimitCPU indicates whether to automatically set the CPU limit to doris BE config. + Default value is 'false'. This means that the Doris BE is unaware of the CPU configuration of resources. + Enabling this configuration means injecting an ENV named BE_CPU_LIMIT with the value requests.cpu into the pod. This configuration will also appear in the 'be.conf' file inside the BE container. + Changing this configuration will cause a BE rolling restart. + type: boolean claims: description: |- Claims lists the names of resources, defined in spec.resourceClaims, diff --git a/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml b/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml index 593db396..c4121592 100644 --- a/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml +++ b/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml @@ -1019,6 +1019,13 @@ spec: Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. type: object + autoResolveLimitCPU: + description: |- + AutoResolveLimitCPU indicates whether to automatically set the CPU limit to doris BE config. + Default value is 'false'. This means that the Doris BE is unaware of the CPU configuration of resources. + Enabling this configuration means injecting an ENV named BE_CPU_LIMIT with the value requests.cpu into the pod. This configuration will also appear in the 'be.conf' file inside the BE container. + Changing this configuration will cause a BE rolling restart. + type: boolean claims: description: |- Claims lists the names of resources, defined in spec.resourceClaims, diff --git a/config/crd/bases/doris.apache.com_dorisclusters.yaml b/config/crd/bases/doris.apache.com_dorisclusters.yaml index 0f2de7de..d700e293 100644 --- a/config/crd/bases/doris.apache.com_dorisclusters.yaml +++ b/config/crd/bases/doris.apache.com_dorisclusters.yaml @@ -1008,6 +1008,13 @@ spec: description: annotation for fe pods. user can config monitor annotation for collect to monitor system. type: object + autoResolveLimitCPU: + description: |- + AutoResolveLimitCPU indicates whether to automatically set the CPU limit to doris BE config. + Default value is 'false'. This means that the Doris BE is unaware of the CPU configuration of resources. + Enabling this configuration means injecting an ENV named BE_CPU_LIMIT with the value requests.cpu into the pod. This configuration will also appear in the 'be.conf' file inside the BE container. + Changing this configuration will cause a BE rolling restart. + type: boolean claims: description: |- Claims lists the names of resources, defined in spec.resourceClaims, @@ -5093,6 +5100,13 @@ spec: description: annotation for fe pods. user can config monitor annotation for collect to monitor system. type: object + autoResolveLimitCPU: + description: |- + AutoResolveLimitCPU indicates whether to automatically set the CPU limit to doris BE config. + Default value is 'false'. This means that the Doris BE is unaware of the CPU configuration of resources. + Enabling this configuration means injecting an ENV named BE_CPU_LIMIT with the value requests.cpu into the pod. This configuration will also appear in the 'be.conf' file inside the BE container. + Changing this configuration will cause a BE rolling restart. + type: boolean autoScalingPolicy: description: AutoScalingPolicy auto scaling strategy properties: diff --git a/config/crd/bases/doris.selectdb.com_dorisclusters.yaml b/config/crd/bases/doris.selectdb.com_dorisclusters.yaml index 0f2de7de..d700e293 100644 --- a/config/crd/bases/doris.selectdb.com_dorisclusters.yaml +++ b/config/crd/bases/doris.selectdb.com_dorisclusters.yaml @@ -1008,6 +1008,13 @@ spec: description: annotation for fe pods. user can config monitor annotation for collect to monitor system. type: object + autoResolveLimitCPU: + description: |- + AutoResolveLimitCPU indicates whether to automatically set the CPU limit to doris BE config. + Default value is 'false'. This means that the Doris BE is unaware of the CPU configuration of resources. + Enabling this configuration means injecting an ENV named BE_CPU_LIMIT with the value requests.cpu into the pod. This configuration will also appear in the 'be.conf' file inside the BE container. + Changing this configuration will cause a BE rolling restart. + type: boolean claims: description: |- Claims lists the names of resources, defined in spec.resourceClaims, @@ -5093,6 +5100,13 @@ spec: description: annotation for fe pods. user can config monitor annotation for collect to monitor system. type: object + autoResolveLimitCPU: + description: |- + AutoResolveLimitCPU indicates whether to automatically set the CPU limit to doris BE config. + Default value is 'false'. This means that the Doris BE is unaware of the CPU configuration of resources. + Enabling this configuration means injecting an ENV named BE_CPU_LIMIT with the value requests.cpu into the pod. This configuration will also appear in the 'be.conf' file inside the BE container. + Changing this configuration will cause a BE rolling restart. + type: boolean autoScalingPolicy: description: AutoScalingPolicy auto scaling strategy properties: diff --git a/go.sum b/go.sum index 645fe1aa..01063fdd 100644 --- a/go.sum +++ b/go.sum @@ -379,8 +379,7 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/helm-charts/doris-operator/crds/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml b/helm-charts/doris-operator/crds/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml index 593db396..c4121592 100644 --- a/helm-charts/doris-operator/crds/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml +++ b/helm-charts/doris-operator/crds/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml @@ -1019,6 +1019,13 @@ spec: Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. type: object + autoResolveLimitCPU: + description: |- + AutoResolveLimitCPU indicates whether to automatically set the CPU limit to doris BE config. + Default value is 'false'. This means that the Doris BE is unaware of the CPU configuration of resources. + Enabling this configuration means injecting an ENV named BE_CPU_LIMIT with the value requests.cpu into the pod. This configuration will also appear in the 'be.conf' file inside the BE container. + Changing this configuration will cause a BE rolling restart. + type: boolean claims: description: |- Claims lists the names of resources, defined in spec.resourceClaims, diff --git a/helm-charts/doris-operator/crds/doris.apache.com_dorisclusters.yaml b/helm-charts/doris-operator/crds/doris.apache.com_dorisclusters.yaml index 0f2de7de..d700e293 100644 --- a/helm-charts/doris-operator/crds/doris.apache.com_dorisclusters.yaml +++ b/helm-charts/doris-operator/crds/doris.apache.com_dorisclusters.yaml @@ -1008,6 +1008,13 @@ spec: description: annotation for fe pods. user can config monitor annotation for collect to monitor system. type: object + autoResolveLimitCPU: + description: |- + AutoResolveLimitCPU indicates whether to automatically set the CPU limit to doris BE config. + Default value is 'false'. This means that the Doris BE is unaware of the CPU configuration of resources. + Enabling this configuration means injecting an ENV named BE_CPU_LIMIT with the value requests.cpu into the pod. This configuration will also appear in the 'be.conf' file inside the BE container. + Changing this configuration will cause a BE rolling restart. + type: boolean claims: description: |- Claims lists the names of resources, defined in spec.resourceClaims, @@ -5093,6 +5100,13 @@ spec: description: annotation for fe pods. user can config monitor annotation for collect to monitor system. type: object + autoResolveLimitCPU: + description: |- + AutoResolveLimitCPU indicates whether to automatically set the CPU limit to doris BE config. + Default value is 'false'. This means that the Doris BE is unaware of the CPU configuration of resources. + Enabling this configuration means injecting an ENV named BE_CPU_LIMIT with the value requests.cpu into the pod. This configuration will also appear in the 'be.conf' file inside the BE container. + Changing this configuration will cause a BE rolling restart. + type: boolean autoScalingPolicy: description: AutoScalingPolicy auto scaling strategy properties: diff --git a/pkg/common/utils/resource/pod.go b/pkg/common/utils/resource/pod.go index cdc93242..53dd1076 100644 --- a/pkg/common/utils/resource/pod.go +++ b/pkg/common/utils/resource/pod.go @@ -18,6 +18,9 @@ package resource import ( + "strconv" + "strings" + dv1 "github.com/apache/doris-operator/api/disaggregated/v1" v1 "github.com/apache/doris-operator/api/doris/v1" "github.com/apache/doris-operator/pkg/common/utils/kerberos" @@ -26,8 +29,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/klog/v2" - "strconv" - "strings" ) const ( @@ -87,7 +88,7 @@ const ( DISAGGREGATED_FE_MAIN_CONTAINER_NAME = "fe" DISAGGREGATED_BE_MAIN_CONTAINER_NAME = "compute" - DISAGGREGATED_MS_MAIN_CONTAINER_NAME= "metaservice" + DISAGGREGATED_MS_MAIN_CONTAINER_NAME = "metaservice" ) type ProbeType string @@ -452,6 +453,14 @@ func NewBaseMainContainer(dcr *v1.DorisCluster, config map[string]interface{}, c envs = append(envs, corev1.EnvVar{Name: "SKIP_CHECK_ULIMIT", Value: "true"}) } + if componentType == v1.Component_BE && dcr.Spec.BeSpec.AutoResolveLimitCPU && spec.Limits.Cpu() != nil { + envs = append(envs, corev1.EnvVar{Name: "BE_CPU_LIMIT", Value: spec.Limits.Cpu().String()}) + } + + if componentType == v1.Component_CN && dcr.Spec.CnSpec.AutoResolveLimitCPU && spec.Limits.Cpu() != nil { + envs = append(envs, corev1.EnvVar{Name: "BE_CPU_LIMIT", Value: spec.Limits.Cpu().String()}) + } + if len(GetMountConfigMapInfo(spec.ConfigMapInfo)) != 0 { _, configVolumeMounts := getMultiConfigVolumeAndVolumeMount(&spec.ConfigMapInfo, componentType) volumeMounts = append(volumeMounts, configVolumeMounts...) @@ -571,12 +580,12 @@ func buildKerberosEnv(info *v1.KerberosInfo, config map[string]interface{}, comp return buildKerberosEnvUseSecretMountPath(info.KeytabPath, config, string(componentType)) } -func BuildKerberosEnvForDDC(info *dv1.KerberosInfo, config map[string]interface{}, componentType dv1.DisaggregatedComponentType)[]corev1.EnvVar { +func BuildKerberosEnvForDDC(info *dv1.KerberosInfo, config map[string]interface{}, componentType dv1.DisaggregatedComponentType) []corev1.EnvVar { if info == nil { return nil } - return buildKerberosEnvUseSecretMountPath(info.KeytabPath, config, string(componentType)) + return buildKerberosEnvUseSecretMountPath(info.KeytabPath, config, string(componentType)) } func buildKerberosEnvUseSecretMountPath(keytabPath string, config map[string]interface{}, componentType string) []corev1.EnvVar { @@ -739,8 +748,7 @@ func getFeDefaultVolumesVolumeMounts() ([]corev1.Volume, []corev1.VolumeMount) { return volumes, volumMounts } -// -func GetPodInfoVolumesVolumeMounts() ([]corev1.Volume, []corev1.VolumeMount){ +func GetPodInfoVolumesVolumeMounts() ([]corev1.Volume, []corev1.VolumeMount) { return appendPodInfoVolumesVolumeMounts(nil, nil) } @@ -932,8 +940,8 @@ func getKerberosVolumeAndVolumeMount(kerberosInfo *v1.KerberosInfo) ([]corev1.Vo return getKerberosConfigAndSecretVolumeAndVolumeMount(kerberosInfo.Krb5ConfigMap, kerberosInfo.KeytabSecretName) } -//get the kerberos volume and mounts to ddc. -func GetDv1KerberosVolumeAndVolumeMount(kerberosInfo *dv1.KerberosInfo)([]corev1.Volume, []corev1.VolumeMount) { +// get the kerberos volume and mounts to ddc. +func GetDv1KerberosVolumeAndVolumeMount(kerberosInfo *dv1.KerberosInfo) ([]corev1.Volume, []corev1.VolumeMount) { if kerberosInfo == nil { return []corev1.Volume{}, []corev1.VolumeMount{} } @@ -941,7 +949,7 @@ func GetDv1KerberosVolumeAndVolumeMount(kerberosInfo *dv1.KerberosInfo)([]corev1 return getKerberosConfigAndSecretVolumeAndVolumeMount(kerberosInfo.Krb5ConfigMap, kerberosInfo.KeytabSecretName) } -//abstract a base function for dcr and ddc used. +// abstract a base function for dcr and ddc used. func getKerberosConfigAndSecretVolumeAndVolumeMount(configMapName, secretName string) ([]corev1.Volume, []corev1.VolumeMount) { var volumes []corev1.Volume var volumeMounts []corev1.VolumeMount @@ -1226,7 +1234,7 @@ func constructBeDefaultInitContainer(defaultImage string) corev1.Container { func UseNewDefaultInitContainerImage(pts *corev1.PodTemplateSpec) { var beImage string for _, c := range pts.Spec.Containers { - if c.Name == string(v1.Component_BE) || c.Name == string(DISAGGREGATED_BE_MAIN_CONTAINER_NAME){ + if c.Name == string(v1.Component_BE) || c.Name == string(DISAGGREGATED_BE_MAIN_CONTAINER_NAME) { beImage = c.Image break } diff --git a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go index 9bfbaff4..1d8dc359 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go @@ -19,6 +19,8 @@ package computegroups import ( "fmt" + "strconv" + dv1 "github.com/apache/doris-operator/api/disaggregated/v1" "github.com/apache/doris-operator/pkg/common/utils/resource" sub "github.com/apache/doris-operator/pkg/controller/sub_controller" @@ -26,7 +28,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" - "strconv" ) const ( @@ -150,6 +151,10 @@ func (dcgs *DisaggregatedComputeGroupsController) NewCGContainer(ddc *dv1.DorisD c.Env = append(c.Env, corev1.EnvVar{Name: "SKIP_CHECK_ULIMIT", Value: "true"}) } + if cg.AutoResolveLimitCPU && cg.Limits.Cpu() != nil { + c.Env = append(c.Env, corev1.EnvVar{Name: "BE_CPU_LIMIT", Value: cg.Limits.Cpu().String()}) + } + resource.BuildDisaggregatedProbe(&c, &cg.CommonSpec, dv1.DisaggregatedBE) _, vms, _ := dcgs.BuildVolumesVolumeMountsAndPVCs(cvs, dv1.DisaggregatedBE, &cg.CommonSpec) _, cmvms := dcgs.BuildDefaultConfigMapVolumesVolumeMounts(cg.ConfigMaps) @@ -210,6 +215,6 @@ func (dcgs *DisaggregatedComputeGroupsController) newSpecificEnvs(ddc *dv1.Doris return cgEnvs } -func(dcgs *DisaggregatedComputeGroupsController) useNewDefaultValuesInStatefulset(st *appv1.StatefulSet) { +func (dcgs *DisaggregatedComputeGroupsController) useNewDefaultValuesInStatefulset(st *appv1.StatefulSet) { resource.UseNewDefaultInitContainerImage(&st.Spec.Template) -} \ No newline at end of file +}