feat: add resource-manager support to PD microservices (#6635)

Author: okJiang

api/core/v1alpha1/common_types.go

@@ -126,15 +126,16 @@ const (
 
 const (
 	// Label value for meta.LabelKeyComponent
-	LabelValComponentPD         = string(meta.ComponentPD)
-	LabelValComponentTiDB       = string(meta.ComponentTiDB)
-	LabelValComponentTiKV       = string(meta.ComponentTiKV)
-	LabelValComponentTiKVWorker = string(meta.ComponentTiKVWorker)
-	LabelValComponentTiFlash    = string(meta.ComponentTiFlash)
-	LabelValComponentTiCDC      = string(meta.ComponentTiCDC)
-	LabelValComponentTSO        = string(meta.ComponentTSO)
-	LabelValComponentScheduling = string(meta.ComponentScheduling)
-	LabelValComponentTiProxy    = string(meta.ComponentTiProxy)
+	LabelValComponentPD              = string(meta.ComponentPD)
+	LabelValComponentTiDB            = string(meta.ComponentTiDB)
+	LabelValComponentTiKV            = string(meta.ComponentTiKV)
+	LabelValComponentTiKVWorker      = string(meta.ComponentTiKVWorker)
+	LabelValComponentTiFlash         = string(meta.ComponentTiFlash)
+	LabelValComponentTiCDC           = string(meta.ComponentTiCDC)
+	LabelValComponentTSO             = string(meta.ComponentTSO)
+	LabelValComponentScheduling      = string(meta.ComponentScheduling)
+	LabelValComponentResourceManager = string(meta.ComponentResourceManager)
+	LabelValComponentTiProxy         = string(meta.ComponentTiProxy)
 	// Deprecated: use LabelValComponentScheduling
 	LabelValComponentScheduler = string(meta.ComponentScheduler)
 

api/core/v1alpha1/names.go

@@ -56,15 +56,16 @@ const (
 const (
 	// Main component containers of the tidb cluster
 	// These names are well known so the name prefix is not added.
-	ContainerNamePD         = "pd"
-	ContainerNameTiKV       = "tikv"
-	ContainerNameTiKVWorker = "tikv-worker"
-	ContainerNameTiDB       = "tidb"
-	ContainerNameTiFlash    = "tiflash"
-	ContainerNameTiCDC      = "ticdc"
-	ContainerNameTSO        = "tso"
-	ContainerNameScheduling = "scheduling"
-	ContainerNameTiProxy    = "tiproxy"
+	ContainerNamePD              = "pd"
+	ContainerNameTiKV            = "tikv"
+	ContainerNameTiKVWorker      = "tikv-worker"
+	ContainerNameTiDB            = "tidb"
+	ContainerNameTiFlash         = "tiflash"
+	ContainerNameTiCDC           = "ticdc"
+	ContainerNameTSO             = "tso"
+	ContainerNameScheduling      = "scheduling"
+	ContainerNameResourceManager = "resource-manager"
+	ContainerNameTiProxy         = "tiproxy"
 	// Deprecated: use ContainerNameScheduling
 	ContainerNameScheduler = "scheduler"
 
@@ -92,15 +93,16 @@ const (
 // All well known dir path
 const (
 	// config dir path
-	DirPathConfigPD         = "/etc/pd"
-	DirPathConfigTiKV       = "/etc/tikv"
-	DirPathConfigTiKVWorker = "/etc/tikv-worker"
-	DirPathConfigTiDB       = "/etc/tidb"
-	DirPathConfigTiFlash    = "/etc/tiflash"
-	DirPathConfigTiCDC      = "/etc/ticdc"
-	DirPathConfigTSO        = "/etc/tso"
-	DirPathConfigScheduling = "/etc/scheduling"
-	DirPathConfigTiProxy    = "/etc/tiproxy"
+	DirPathConfigPD              = "/etc/pd"
+	DirPathConfigTiKV            = "/etc/tikv"
+	DirPathConfigTiKVWorker      = "/etc/tikv-worker"
+	DirPathConfigTiDB            = "/etc/tidb"
+	DirPathConfigTiFlash         = "/etc/tiflash"
+	DirPathConfigTiCDC           = "/etc/ticdc"
+	DirPathConfigTSO             = "/etc/tso"
+	DirPathConfigScheduling      = "/etc/scheduling"
+	DirPathConfigResourceManager = "/etc/resource-manager"
+	DirPathConfigTiProxy         = "/etc/tiproxy"
 	// Deprecated: use DirPathConfigScheduling
 	DirPathConfigScheduler = "/etc/scheduler"
 
@@ -120,16 +122,17 @@ const (
 	// TLS
 	//
 	// Dir path of cluster tls file
-	DirPathClusterTLSPD         = "/var/lib/pd-tls"
-	DirPathClusterTLSTiKV       = "/var/lib/tikv-tls"
-	DirPathClusterTLSTiKVWorker = "/var/lib/tikv-worker-tls"
-	DirPathClusterTLSTiDB       = "/var/lib/tidb-tls"
-	DirPathClusterTLSTiFlash    = "/var/lib/tiflash-tls"
-	DirPathClusterClientTLS     = "/var/lib/cluster-client-tls"
-	DirPathClusterTLSTiCDC      = "/var/lib/ticdc-tls"
-	DirPathClusterTLSTSO        = "/var/lib/tso-tls"
-	DirPathClusterTLSScheduling = "/var/lib/scheduling-tls"
-	DirPathClusterTLSTiProxy    = "/var/lib/tiproxy-tls"
+	DirPathClusterTLSPD              = "/var/lib/pd-tls"
+	DirPathClusterTLSTiKV            = "/var/lib/tikv-tls"
+	DirPathClusterTLSTiKVWorker      = "/var/lib/tikv-worker-tls"
+	DirPathClusterTLSTiDB            = "/var/lib/tidb-tls"
+	DirPathClusterTLSTiFlash         = "/var/lib/tiflash-tls"
+	DirPathClusterClientTLS          = "/var/lib/cluster-client-tls"
+	DirPathClusterTLSTiCDC           = "/var/lib/ticdc-tls"
+	DirPathClusterTLSTSO             = "/var/lib/tso-tls"
+	DirPathClusterTLSScheduling      = "/var/lib/scheduling-tls"
+	DirPathClusterTLSResourceManager = "/var/lib/resource-manager-tls"
+	DirPathClusterTLSTiProxy         = "/var/lib/tiproxy-tls"
 	// Deprecated: use DirPathClusterTLSScheduling
 	DirPathClusterTLSScheduler = "/var/lib/scheduler-tls"
 

api/core/v1alpha1/resourcemanager_types.go

@@ -0,0 +1,206 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	meta "github.com/pingcap/tidb-operator/api/v2/meta/v1alpha1"
+)
+
+const (
+	ResourceManagerPortNameClient    = "client"
+	DefaultResourceManagerPortClient = 3379
+
+	// DefaultResourceManagerMinReadySeconds is default min ready seconds of resource manager
+	DefaultResourceManagerMinReadySeconds = 5
+)
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +kubebuilder:object:root=true
+
+// ResourceManagerGroupList defines a list of ResourceManager groups
+type ResourceManagerGroupList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+
+	Items []ResourceManagerGroup `json:"items"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.selector
+// +kubebuilder:resource:categories=group,shortName=rmg
+// +kubebuilder:selectablefield:JSONPath=`.spec.cluster.name`
+// +kubebuilder:printcolumn:name="Cluster",type=string,JSONPath=`.spec.cluster.name`
+// +kubebuilder:printcolumn:name="Desired",type=string,JSONPath=`.spec.replicas`
+// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.readyReplicas`
+// +kubebuilder:printcolumn:name="Updated",type=string,JSONPath=`.status.updatedReplicas`
+// +kubebuilder:printcolumn:name="UpdateRevision",type=string,JSONPath=`.status.updateRevision`
+// +kubebuilder:printcolumn:name="CurrentRevision",type=string,JSONPath=`.status.currentRevision`
+// +kubebuilder:printcolumn:name="Synced",type=string,JSONPath=`.status.conditions[?(@.type=="Synced")].status`
+// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status`
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
+
+// ResourceManagerGroup defines a group of similar ResourceManager instances
+// +kubebuilder:validation:XValidation:rule="size(self.metadata.name) <= 40",message="name must not exceed 40 characters"
+type ResourceManagerGroup struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   ResourceManagerGroupSpec   `json:"spec,omitempty"`
+	Status ResourceManagerGroupStatus `json:"status,omitempty"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +kubebuilder:object:root=true
+
+// ResourceManagerList defines a list of ResourceManager instances
+type ResourceManagerList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+
+	Items []ResourceManager `json:"items"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:categories=instance
+// +kubebuilder:selectablefield:JSONPath=`.spec.cluster.name`
+// +kubebuilder:printcolumn:name="Cluster",type=string,JSONPath=`.spec.cluster.name`
+// +kubebuilder:printcolumn:name="Synced",type=string,JSONPath=`.status.conditions[?(@.type=="Synced")].status`
+// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status`
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
+
+// ResourceManager defines a ResourceManager instance
+// +kubebuilder:validation:XValidation:rule="size(self.metadata.name) <= 47",message="name must not exceed 47 characters"
+type ResourceManager struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   ResourceManagerSpec   `json:"spec,omitempty"`
+	Status ResourceManagerStatus `json:"status,omitempty"`
+}
+
+// ResourceManagerGroupSpec describes the common attributes of a ResourceManagerGroup
+
+type ResourceManagerGroupSpec struct {
+	Cluster ClusterReference `json:"cluster"`
+	// Features are enabled feature
+	Features []meta.Feature `json:"features,omitempty"`
+
+	// +kubebuilder:validation:Minimum=0
+	Replicas *int32 `json:"replicas"`
+
+	// +listType=map
+	// +listMapKey=type
+	SchedulePolicies []SchedulePolicy `json:"schedulePolicies,omitempty"`
+
+	// MinReadySeconds specifies the minimum number of seconds for which a newly created pod be ready without any of its containers crashing, for it to be considered available.
+	// +kubebuilder:validation:Minimum=0
+	// +optional
+	MinReadySeconds *int64 `json:"minReadySeconds,omitempty"`
+
+	Template ResourceManagerTemplate `json:"template"`
+}
+
+type ResourceManagerTemplate struct {
+	ObjectMeta `json:"metadata,omitempty"`
+	Spec       ResourceManagerTemplateSpec `json:"spec"`
+}
+
+// ResourceManagerTemplateSpec can only be specified in ResourceManagerGroup
+// TODO: It's name may need to be changed to distinguish from PodTemplateSpec
+// +kubebuilder:validation:XValidation:rule="!has(self.overlay) || !has(self.overlay.volumeClaims) || (has(self.volumes) && self.overlay.volumeClaims.all(vc, vc.name in self.volumes.map(v, v.name)))",message="overlay volumeClaims names must exist in volumes"
+
+type ResourceManagerTemplateSpec struct {
+	// Version must be a semantic version.
+	// It can has a v prefix or not.
+	// +kubebuilder:validation:Pattern=`^(v)?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`
+	Version string `json:"version"`
+
+	// Image is pd's image
+	// If tag is omitted, version will be used as the image tag.
+	// Default is pingcap/pd
+	Image *string `json:"image,omitempty"`
+
+	// Server defines server config for ResourceManager
+	Server ResourceManagerServer `json:"server,omitempty"`
+
+	Resources ResourceRequirements `json:"resources,omitempty"`
+
+	UpdateStrategy UpdateStrategy `json:"updateStrategy,omitempty"`
+
+	// Config defines config file of ResourceManager
+	Config ConfigFile `json:"config,omitempty"`
+
+	// Security defines security config
+	Security *Security `json:"security,omitempty"`
+
+	// Volumes defines persistent volumes of ResourceManager
+	// +listType=map
+	// +listMapKey=name
+	// +kubebuilder:validation:MaxItems=256
+	Volumes []Volume `json:"volumes,omitempty"`
+
+	// Overlay defines a k8s native resource template patch
+	// All resources(pod, pvcs, ...) managed by ResourceManager can be overlayed by this field
+	Overlay *Overlay `json:"overlay,omitempty"`
+}
+
+type ResourceManagerServer struct {
+	// Ports defines all ports listened by ResourceManager
+	Ports ResourceManagerPorts `json:"ports,omitempty"`
+}
+
+type ResourceManagerPorts struct {
+	// Client defines port for ResourceManager's api service
+	Client *Port `json:"client,omitempty"`
+}
+
+type ResourceManagerGroupStatus struct {
+	CommonStatus `json:",inline"`
+	GroupStatus  `json:",inline"`
+}
+
+// ResourceManagerSpec describes the common attributes of a ResourceManager instance
+// +kubebuilder:validation:XValidation:rule="(!has(oldSelf.topology) && !has(self.topology)) || (has(oldSelf.topology) && has(self.topology))",fieldPath=".topology",message="topology can only be set when creating"
+
+type ResourceManagerSpec struct {
+	// Cluster is a reference of tidb cluster
+	Cluster ClusterReference `json:"cluster"`
+	// Features are enabled feature
+	Features []meta.Feature `json:"features,omitempty"`
+
+	// Topology defines the topology domain of this ResourceManager instance
+	// It will be translated into a node affinity config
+	// Topology cannot be changed
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="topology is immutable"
+	Topology Topology `json:"topology,omitempty"`
+
+	// Subdomain means the subdomain of the exported ResourceManager dns.
+	// A same ResourceManager cluster will use a same subdomain
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="subdomain is immutable"
+	Subdomain string `json:"subdomain"`
+
+	// ResourceManagerTemplateSpec embedded some fields managed by ResourceManagerGroup
+	ResourceManagerTemplateSpec `json:",inline"`
+}
+
+type ResourceManagerStatus struct {
+	CommonStatus `json:",inline"`
+}