diff --git a/api/constants/constants.go b/api/constants/constants.go index ef312238..6d8747f8 100644 --- a/api/constants/constants.go +++ b/api/constants/constants.go @@ -24,6 +24,11 @@ const ( // OnboardingNamespaceLabel is used to store the namespace on the onboarding cluster of a resource. OnboardingNamespaceLabel = OpenMCPGroupName + "/onboarding-namespace" + // TopologyLabel is used to indicate the topology name of the resource. + TopologyLabel = OpenMCPGroupName + "/topology" + // TopologyNamespaceLabel is used to indicate the topology namespace of the resource. + TopologyNamespaceLabel = TopologyLabel + "-ns" + // EnvVariablePodName is the name of an environment variable passed to providers. // Its value is the name of the pod in which the provider is running. EnvVariablePodName = "POD_NAME" diff --git a/api/crds/manifests/openmcp.cloud_clusterproviders.yaml b/api/crds/manifests/openmcp.cloud_clusterproviders.yaml index 47e17a94..6a84d342 100644 --- a/api/crds/manifests/openmcp.cloud_clusterproviders.yaml +++ b/api/crds/manifests/openmcp.cloud_clusterproviders.yaml @@ -2064,6 +2064,196 @@ spec: items: type: string type: array + runReplicas: + default: 1 + description: |- + RunReplicas is the number of replicas for the provider controller. + Defaults to 1. + If greater thant 1, automatically sets the `--leader-elect=true` flag in the RunCommand. + format: int32 + minimum: 1 + type: integer + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how to spread the provider pods + across your cluster among failure-domains such as zones, nodes, regions, etc. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + The label selector for the topology spread constraints is automatically set to match the provider deployment pods. + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + x-kubernetes-list-type: map verbosity: default: INFO description: Verbosity is the verbosity level of the provider. diff --git a/api/crds/manifests/openmcp.cloud_platformservices.yaml b/api/crds/manifests/openmcp.cloud_platformservices.yaml index 3aad4bcb..9b5b8575 100644 --- a/api/crds/manifests/openmcp.cloud_platformservices.yaml +++ b/api/crds/manifests/openmcp.cloud_platformservices.yaml @@ -2064,6 +2064,196 @@ spec: items: type: string type: array + runReplicas: + default: 1 + description: |- + RunReplicas is the number of replicas for the provider controller. + Defaults to 1. + If greater thant 1, automatically sets the `--leader-elect=true` flag in the RunCommand. + format: int32 + minimum: 1 + type: integer + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how to spread the provider pods + across your cluster among failure-domains such as zones, nodes, regions, etc. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + The label selector for the topology spread constraints is automatically set to match the provider deployment pods. + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + x-kubernetes-list-type: map verbosity: default: INFO description: Verbosity is the verbosity level of the provider. diff --git a/api/crds/manifests/openmcp.cloud_serviceproviders.yaml b/api/crds/manifests/openmcp.cloud_serviceproviders.yaml index 3b11e566..f870c747 100644 --- a/api/crds/manifests/openmcp.cloud_serviceproviders.yaml +++ b/api/crds/manifests/openmcp.cloud_serviceproviders.yaml @@ -2064,6 +2064,196 @@ spec: items: type: string type: array + runReplicas: + default: 1 + description: |- + RunReplicas is the number of replicas for the provider controller. + Defaults to 1. + If greater thant 1, automatically sets the `--leader-elect=true` flag in the RunCommand. + format: int32 + minimum: 1 + type: integer + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how to spread the provider pods + across your cluster among failure-domains such as zones, nodes, regions, etc. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + The label selector for the topology spread constraints is automatically set to match the provider deployment pods. + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + x-kubernetes-list-type: map verbosity: default: INFO description: Verbosity is the verbosity level of the provider. diff --git a/api/provider/v1alpha1/deployment_types.go b/api/provider/v1alpha1/deployment_types.go index 91dfa671..e36193ea 100644 --- a/api/provider/v1alpha1/deployment_types.go +++ b/api/provider/v1alpha1/deployment_types.go @@ -75,6 +75,25 @@ type DeploymentSpec struct { // +listType=map // +listMapKey=mountPath ExtraVolumeMounts []corev1.VolumeMount `json:"extraVolumeMounts,omitempty" patchStrategy:"merge" patchMergeKey:"mountPath"` + + // RunReplicas is the number of replicas for the provider controller. + // Defaults to 1. + // If greater thant 1, automatically sets the `--leader-elect=true` flag in the RunCommand. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:default=1 + // +optional + RunReplicas int32 `json:"runReplicas,omitempty"` + + // TopologySpreadConstraints describes how to spread the provider pods + // across your cluster among failure-domains such as zones, nodes, regions, etc. + // More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + // The label selector for the topology spread constraints is automatically set to match the provider deployment pods. + // +optional + // +patchMergeKey=topologyKey + // +patchStrategy=merge + // +listType=map + // +listMapKey=topologyKey + TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty" patchStrategy:"merge" patchMergeKey:"topologyKey"` } // DeploymentStatus defines the observed state of a provider. diff --git a/api/provider/v1alpha1/zz_generated.deepcopy.go b/api/provider/v1alpha1/zz_generated.deepcopy.go index b3295d53..c4826b5e 100644 --- a/api/provider/v1alpha1/zz_generated.deepcopy.go +++ b/api/provider/v1alpha1/zz_generated.deepcopy.go @@ -140,6 +140,13 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.TopologySpreadConstraints != nil { + in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints + *out = make([]v1.TopologySpreadConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentSpec. diff --git a/cmd/openmcp-operator/app/mcp/run.go b/cmd/openmcp-operator/app/mcp/run.go index 80448f3c..c766be2b 100644 --- a/cmd/openmcp-operator/app/mcp/run.go +++ b/cmd/openmcp-operator/app/mcp/run.go @@ -269,14 +269,20 @@ func (o *RunOptions) Run(ctx context.Context) error { TLSOpts: o.WebhookTLSOpts, }) + podNamespace := os.Getenv(apiconst.EnvVariablePodNamespace) + if podNamespace == "" { + return fmt.Errorf("environment variable %s is not set", apiconst.EnvVariablePodNamespace) + } + mgr, err := ctrl.NewManager(onboardingCluster.RESTConfig(), ctrl.Options{ - Scheme: onboardingCluster.Scheme(), - Metrics: o.MetricsServerOptions, - WebhookServer: webhookServer, - HealthProbeBindAddress: o.ProbeAddr, - PprofBindAddress: o.PprofAddr, - LeaderElection: o.EnableLeaderElection, - LeaderElectionID: "github.com/openmcp-project/openmcp-operator--mcp-controller", + Scheme: onboardingCluster.Scheme(), + Metrics: o.MetricsServerOptions, + WebhookServer: webhookServer, + HealthProbeBindAddress: o.ProbeAddr, + PprofBindAddress: o.PprofAddr, + LeaderElection: o.EnableLeaderElection, + LeaderElectionID: "github.com/openmcp-project/openmcp-operator--mcp-controller", + LeaderElectionNamespace: podNamespace, // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily // when the Manager ends. This requires the binary to immediately end when the // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly diff --git a/cmd/openmcp-operator/app/run.go b/cmd/openmcp-operator/app/run.go index 1f462106..e5966343 100644 --- a/cmd/openmcp-operator/app/run.go +++ b/cmd/openmcp-operator/app/run.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "fmt" + "os" "path/filepath" "slices" "strings" @@ -23,6 +24,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/yaml" + apiconst "github.com/openmcp-project/openmcp-operator/api/constants" + "github.com/openmcp-project/openmcp-operator/api/install" "github.com/openmcp-project/openmcp-operator/api/provider/v1alpha1" "github.com/openmcp-project/openmcp-operator/cmd/openmcp-operator/app/options" @@ -261,14 +264,20 @@ func (o *RunOptions) Run(ctx context.Context) error { TLSOpts: o.WebhookTLSOpts, }) + podNamespace := os.Getenv(apiconst.EnvVariablePodNamespace) + if podNamespace == "" { + return fmt.Errorf("environment variable %s is not set", apiconst.EnvVariablePodNamespace) + } + mgr, err := ctrl.NewManager(o.PlatformCluster.RESTConfig(), ctrl.Options{ - Scheme: install.InstallOperatorAPIsPlatform(runtime.NewScheme()), - Metrics: o.MetricsServerOptions, - WebhookServer: webhookServer, - HealthProbeBindAddress: o.ProbeAddr, - PprofBindAddress: o.PprofAddr, - LeaderElection: o.EnableLeaderElection, - LeaderElectionID: "github.com/openmcp-project/openmcp-operator", + Scheme: install.InstallOperatorAPIsPlatform(runtime.NewScheme()), + Metrics: o.MetricsServerOptions, + WebhookServer: webhookServer, + HealthProbeBindAddress: o.ProbeAddr, + PprofBindAddress: o.PprofAddr, + LeaderElection: o.EnableLeaderElection, + LeaderElectionID: "github.com/openmcp-project/openmcp-operator", + LeaderElectionNamespace: podNamespace, // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily // when the Manager ends. This requires the binary to immediately end when the // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly diff --git a/docs/controller/deployment.md b/docs/controller/deployment.md index e4d5ac7e..054cc35c 100644 --- a/docs/controller/deployment.md +++ b/docs/controller/deployment.md @@ -59,9 +59,21 @@ spec: - name: value: verbosity: + runReplicas: 3 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway ``` - The `image` field specifies the container image to use for the init job and deployment of the provider. - The `imagePullSecrets` field specifies a list of secrets that contain the credentials to pull the image from a registry. - The `env` field specifies a list of name-value pairs that are passed as environment variables to the init job and deployment of the provider. - The `verbosity` field specifies the logging level. Supported values are DEBUG, INFO, and ERROR. The default is INFO. +- The `runReplicas` field specifies the number of replicas for the deployment of the provider. The default is `1`. + If set to greater than `1`, the `--leader-elect=true` argument is automatically added to the provider's command line to enable leader election among the replicas. +- The `topologySpreadConstraints` field specifies a list of topology spread constraints for the deployment of the provider. For more information, see the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/). + The label selectors for the topology spread constraints are automatically set to match the pods of the deployment. \ No newline at end of file diff --git a/internal/controllers/provider/controller_test.go b/internal/controllers/provider/controller_test.go index 09e5f854..3a535c0f 100644 --- a/internal/controllers/provider/controller_test.go +++ b/internal/controllers/provider/controller_test.go @@ -11,6 +11,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/openmcp-project/openmcp-operator/api/constants" + commonapi "github.com/openmcp-project/openmcp-operator/api/common" apiinstall "github.com/openmcp-project/openmcp-operator/api/install" "github.com/openmcp-project/openmcp-operator/api/provider/v1alpha1" @@ -97,11 +99,30 @@ var _ = Describe("Deployment Controller", func() { Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(5), "Deployment container should have an environment variables") Expect(deploy.Spec.Template.Spec.Containers[0].Env[0].Name).To(Equal("NAME"), "Deployment container environment variable name should match the provider spec") Expect(deploy.Spec.Template.Spec.Containers[0].Env[0].Value).To(Equal("test-name"), "Deployment container environment variable value should match the provider spec") + Expect(deploy.Spec.Replicas).To(Equal(&deploymentSpec.RunReplicas), "Deployment replicas should match the provider spec") + + if deploymentSpec.RunReplicas > 1 { + Expect(deploy.Spec.Template.Spec.Containers[0].Args).To(ContainElement("--leader-elect=true"), "Deployment container args should contain the leader-elect flag") + } + + if len(deploymentSpec.TopologySpreadConstraints) > 0 { + Expect(deploy.Spec.Template.Labels).To(HaveKeyWithValue(constants.TopologyLabel, deploy.Name), "Deployment pod template labels should contain the topology label") + Expect(deploy.Spec.Template.Labels).To(HaveKeyWithValue(constants.TopologyNamespaceLabel, deploy.Namespace), "Deployment pod template labels should contain the topology namespace label") + + Expect(deploy.Spec.Template.Spec.TopologySpreadConstraints).To(HaveLen(len(deploymentSpec.TopologySpreadConstraints)), "Deployment should have the correct number of topology spread constraints") + + for i, c := range deploymentSpec.TopologySpreadConstraints { + Expect(deploy.Spec.Template.Spec.TopologySpreadConstraints[i].MaxSkew).To(Equal(c.MaxSkew), "Deployment topology spread constraint MaxSkew should match the provider spec") + Expect(deploy.Spec.Template.Spec.TopologySpreadConstraints[i].WhenUnsatisfiable).To(Equal(c.WhenUnsatisfiable), "Deployment topology spread constraint WhenUnsatisfiable should match the provider spec") + Expect(deploy.Spec.Template.Spec.TopologySpreadConstraints[i].LabelSelector.MatchLabels).To(HaveKeyWithValue(constants.TopologyLabel, deploy.Name), "Deployment topology spread constraint LabelSelector should match the provider spec") + Expect(deploy.Spec.Template.Spec.TopologySpreadConstraints[i].LabelSelector.MatchLabels).To(HaveKeyWithValue(constants.TopologyNamespaceLabel, deploy.Namespace), "Deployment topology spread constraint LabelSelector should match the provider spec") + } + } // 3rd reconcile (after deployment is ready) - deploy.Status.Replicas = 1 - deploy.Status.UpdatedReplicas = 1 - deploy.Status.AvailableReplicas = 1 + deploy.Status.Replicas = *deploy.Spec.Replicas + deploy.Status.UpdatedReplicas = *deploy.Spec.Replicas + deploy.Status.AvailableReplicas = *deploy.Spec.Replicas Expect(env.Client().Status().Update(env.Ctx, deploy)).To(Succeed(), "Status update of deployment should succeed") env.ShouldReconcile(req, "3rd reconcile should not return an error") _, deploymentStatus = getProvider(env, provider) @@ -141,6 +162,14 @@ var _ = Describe("Deployment Controller", func() { Expect(deploymentSpec.RunCommand).To(HaveLen(2)) }) + It("should reconcile a service providers with multiple run replicas and topology spread constraints", func() { + env := buildTestEnvironment("test-05", v1alpha1.ServiceProviderGKV()) + req := testutils.RequestFromStrings("service-provider-test-05") + deploymentSpec := reconcileProvider(env, req, v1alpha1.ServiceProviderGKV()) + Expect(deploymentSpec.RunReplicas).To(Equal(int32(3))) + Expect(deploymentSpec.TopologySpreadConstraints).To(HaveLen(2)) + }) + }) Context("Converter", func() { diff --git a/internal/controllers/provider/install/deployment.go b/internal/controllers/provider/install/deployment.go index f251bddf..b6c94cbc 100644 --- a/internal/controllers/provider/install/deployment.go +++ b/internal/controllers/provider/install/deployment.go @@ -11,6 +11,8 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/openmcp-project/openmcp-operator/api/constants" + "github.com/openmcp-project/controller-utils/pkg/resources" "github.com/openmcp-project/openmcp-operator/api/install" @@ -64,8 +66,11 @@ func (m *deploymentMutator) Mutate(d *appsv1.Deployment) error { "--verbosity="+m.values.Verbosity(), "--provider-name="+m.values.provider.GetName(), ) + if m.values.deploymentSpec.RunReplicas > 1 { + runCmd = append(runCmd, "--leader-elect=true") + } d.Spec = appsv1.DeploymentSpec{ - Replicas: ptr.To[int32](1), + Replicas: ptr.To(m.values.deploymentSpec.RunReplicas), Selector: &metav1.LabelSelector{ MatchLabels: m.values.LabelsController(), }, @@ -84,13 +89,29 @@ func (m *deploymentMutator) Mutate(d *appsv1.Deployment) error { VolumeMounts: m.values.deploymentSpec.ExtraVolumeMounts, }, }, - ImagePullSecrets: m.values.ImagePullSecrets(), - ServiceAccountName: m.values.NamespacedDefaultResourceName(), - Volumes: m.values.deploymentSpec.ExtraVolumes, + ImagePullSecrets: m.values.ImagePullSecrets(), + ServiceAccountName: m.values.NamespacedDefaultResourceName(), + Volumes: m.values.deploymentSpec.ExtraVolumes, + TopologySpreadConstraints: m.values.deploymentSpec.TopologySpreadConstraints, }, }, } + if len(m.values.deploymentSpec.TopologySpreadConstraints) > 0 { + for i := range d.Spec.Template.Spec.TopologySpreadConstraints { + labelSelector := &metav1.LabelSelector{ + MatchLabels: map[string]string{ + constants.TopologyLabel: m.values.NamespacedDefaultResourceName(), + constants.TopologyNamespaceLabel: m.values.Namespace(), + }, + } + d.Spec.Template.Spec.TopologySpreadConstraints[i].LabelSelector = labelSelector + } + + d.Spec.Template.Labels[constants.TopologyLabel] = m.values.NamespacedDefaultResourceName() + d.Spec.Template.Labels[constants.TopologyNamespaceLabel] = m.values.Namespace() + } + // Set the provider as owner of the deployment, so that the provider controller gets an event if the deployment changes. if err := controllerutil.SetControllerReference(m.values.provider, d, install.InstallOperatorAPIsPlatform(runtime.NewScheme())); err != nil { return fmt.Errorf("failed to set deployment controller as owner of deployment: %w", err) diff --git a/internal/controllers/provider/testdata/test-01/clusterprovider.yaml b/internal/controllers/provider/testdata/test-01/clusterprovider.yaml index adcf2daf..baca8ee7 100644 --- a/internal/controllers/provider/testdata/test-01/clusterprovider.yaml +++ b/internal/controllers/provider/testdata/test-01/clusterprovider.yaml @@ -10,3 +10,4 @@ spec: - name: NAME value: "test-name" verbosity: DEBUG + runReplicas: 1 \ No newline at end of file diff --git a/internal/controllers/provider/testdata/test-02/serviceprovider.yaml b/internal/controllers/provider/testdata/test-02/serviceprovider.yaml index cd216583..9ea92fe9 100644 --- a/internal/controllers/provider/testdata/test-02/serviceprovider.yaml +++ b/internal/controllers/provider/testdata/test-02/serviceprovider.yaml @@ -10,3 +10,4 @@ spec: - name: NAME value: "test-name" verbosity: DEBUG + runReplicas: 1 \ No newline at end of file diff --git a/internal/controllers/provider/testdata/test-03/platformservice.yaml b/internal/controllers/provider/testdata/test-03/platformservice.yaml index 9fe83019..1a350894 100644 --- a/internal/controllers/provider/testdata/test-03/platformservice.yaml +++ b/internal/controllers/provider/testdata/test-03/platformservice.yaml @@ -10,3 +10,4 @@ spec: - name: NAME value: "test-name" verbosity: DEBUG + runReplicas: 1 \ No newline at end of file diff --git a/internal/controllers/provider/testdata/test-04/platformservice.yaml b/internal/controllers/provider/testdata/test-04/platformservice.yaml index 6ebfb82a..ed903312 100644 --- a/internal/controllers/provider/testdata/test-04/platformservice.yaml +++ b/internal/controllers/provider/testdata/test-04/platformservice.yaml @@ -16,3 +16,4 @@ spec: - name: NAME value: "test-name" verbosity: DEBUG + runReplicas: 1 diff --git a/internal/controllers/provider/testdata/test-05/serviceprovider.yaml b/internal/controllers/provider/testdata/test-05/serviceprovider.yaml new file mode 100644 index 00000000..f6077d1b --- /dev/null +++ b/internal/controllers/provider/testdata/test-05/serviceprovider.yaml @@ -0,0 +1,22 @@ +apiVersion: openmcp.cloud/v1alpha1 +kind: ServiceProvider +metadata: + name: service-provider-test-05 + generation: 1 +spec: + image: "test-image:v0.1.0" + imagePullSecrets: [] + env: + - name: NAME + value: "test-name" + verbosity: DEBUG + + runReplicas: 3 + + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway \ No newline at end of file