Skip to content

Commit b3d4ffa

Browse files
authored
Merge pull request #6933 from baiyutang/feat/support-PDB
add PodDisruptionBudget (PDB) support to the Karmada Operator
2 parents 516dd52 + 6ff54c7 commit b3d4ffa

File tree

16 files changed

+613
-197
lines changed

16 files changed

+613
-197
lines changed

charts/karmada-operator/templates/karmada-operator-clusterrole.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,5 +28,8 @@ rules:
2828
- apiGroups: ["apps"]
2929
resources: ["statefulsets", "deployments"] # to manage statefulsets, e.g. etcd, and deployments, e.g. karmada-operator
3030
verbs: ["get", "create", "update", "delete"]
31+
- apiGroups: ["policy"]
32+
resources: ["poddisruptionbudgets"] # to manage pod disruption budgets for karmada components
33+
verbs: ["get", "create", "update", "delete"]
3134
- nonResourceURLs: ["/healthz"] # used to check whether the karmada apiserver is health
3235
verbs: ["get"]

operator/config/deploy/karmada-operator-clusterrole.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,8 @@ rules:
2929
- apiGroups: ["apps"]
3030
resources: ["statefulsets", "deployments"] # to manage statefulsets, e.g. etcd, and deployments, e.g. karmada-operator
3131
verbs: ["get", "create", "update", "delete"]
32+
- apiGroups: ["policy"]
33+
resources: ["poddisruptionbudgets"] # to manage pod disruption budgets for karmada components
34+
verbs: ["get", "create", "update", "delete"]
3235
- nonResourceURLs: ["/healthz"] # used to check whether the karmada apiserver is health
3336
verbs: ["get"]

operator/pkg/controlplane/apiserver/apiserver.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,21 @@ limitations under the License.
1717
package apiserver
1818

1919
import (
20+
"context"
2021
"fmt"
2122

2223
appsv1 "k8s.io/api/apps/v1"
2324
corev1 "k8s.io/api/core/v1"
25+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2426
"k8s.io/apimachinery/pkg/labels"
2527
kuberuntime "k8s.io/apimachinery/pkg/runtime"
28+
"k8s.io/apimachinery/pkg/runtime/schema"
2629
clientset "k8s.io/client-go/kubernetes"
2730
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
2831

2932
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
3033
"github.com/karmada-io/karmada/operator/pkg/controlplane/etcd"
34+
"github.com/karmada-io/karmada/operator/pkg/controlplane/pdb"
3135
"github.com/karmada-io/karmada/operator/pkg/util"
3236
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
3337
"github.com/karmada-io/karmada/operator/pkg/util/patcher"
@@ -87,6 +91,18 @@ func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.K
8791
if err := apiclient.CreateOrUpdateDeployment(client, apiserverDeployment); err != nil {
8892
return fmt.Errorf("error when creating deployment for %s, err: %w", apiserverDeployment.Name, err)
8993
}
94+
95+
// Fetch persisted Deployment to get real UID
96+
persisted, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), apiserverDeployment.GetName(), metav1.GetOptions{})
97+
if err != nil {
98+
return fmt.Errorf("failed to fetch Deployment %s/%s for PDB owner, err: %w", namespace, apiserverDeployment.GetName(), err)
99+
}
100+
gvk := schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}
101+
ownerRef := *metav1.NewControllerRef(persisted, gvk)
102+
if err := pdb.EnsurePodDisruptionBudget(client, util.KarmadaAPIServerName(name), namespace, cfg.CommonSettings.PodDisruptionBudgetConfig, apiserverDeployment.Spec.Template.Labels, []metav1.OwnerReference{ownerRef}); err != nil {
103+
return fmt.Errorf("failed to ensure PDB for apiserver component %s, err: %w", util.KarmadaAPIServerName(name), err)
104+
}
105+
90106
return nil
91107
}
92108

@@ -158,6 +174,18 @@ func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operator
158174
if err := apiclient.CreateOrUpdateDeployment(client, aggregatedAPIServerDeployment); err != nil {
159175
return fmt.Errorf("error when creating deployment for %s, err: %w", aggregatedAPIServerDeployment.Name, err)
160176
}
177+
178+
// Fetch persisted Deployment to get real UID
179+
persistedAgg, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), aggregatedAPIServerDeployment.GetName(), metav1.GetOptions{})
180+
if err != nil {
181+
return fmt.Errorf("failed to fetch Deployment %s/%s for PDB owner, err: %w", namespace, aggregatedAPIServerDeployment.GetName(), err)
182+
}
183+
gvk2 := schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}
184+
ownerRef2 := *metav1.NewControllerRef(persistedAgg, gvk2)
185+
if err := pdb.EnsurePodDisruptionBudget(client, util.KarmadaAggregatedAPIServerName(name), namespace, cfg.CommonSettings.PodDisruptionBudgetConfig, aggregatedAPIServerDeployment.Spec.Template.Labels, []metav1.OwnerReference{ownerRef2}); err != nil {
186+
return fmt.Errorf("failed to ensure PDB for aggregated apiserver component %s, err: %w", util.KarmadaAggregatedAPIServerName(name), err)
187+
}
188+
161189
return nil
162190
}
163191

operator/pkg/controlplane/apiserver/apiserver_test.go

Lines changed: 91 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,35 @@ func TestEnsureKarmadaAPIServer(t *testing.T) {
6767
}
6868

6969
actions := fakeClient.Actions()
70-
if len(actions) != 2 {
71-
t.Fatalf("expected 2 actions, but got %d", len(actions))
70+
// We now create deployment, service, PDB, and get deployment, so expect 4 actions
71+
if len(actions) != 4 {
72+
t.Fatalf("expected 4 actions, but got %d", len(actions))
73+
}
74+
75+
// Check that we have deployment, service, and PDB
76+
deploymentCount := 0
77+
serviceCount := 0
78+
pdbCount := 0
79+
for _, action := range actions {
80+
if action.GetResource().Resource == "deployments" {
81+
deploymentCount++
82+
} else if action.GetResource().Resource == "services" {
83+
serviceCount++
84+
} else if action.GetResource().Resource == "poddisruptionbudgets" {
85+
pdbCount++
86+
}
87+
}
88+
89+
if deploymentCount != 2 {
90+
t.Errorf("expected 2 deployment actions (create + get), but got %d", deploymentCount)
91+
}
92+
93+
if serviceCount != 1 {
94+
t.Errorf("expected 1 service action, but got %d", serviceCount)
95+
}
96+
97+
if pdbCount != 1 {
98+
t.Errorf("expected 1 PDB action, but got %d", pdbCount)
7299
}
73100
}
74101

@@ -108,8 +135,35 @@ func TestEnsureKarmadaAggregatedAPIServer(t *testing.T) {
108135
}
109136

110137
actions := fakeClient.Actions()
111-
if len(actions) != 2 {
112-
t.Fatalf("expected 2 actions, but got %d", len(actions))
138+
// We now create deployment, service, PDB, and get deployment, so expect 4 actions
139+
if len(actions) != 4 {
140+
t.Fatalf("expected 4 actions, but got %d", len(actions))
141+
}
142+
143+
// Check that we have deployment, service, and PDB
144+
deploymentCount := 0
145+
serviceCount := 0
146+
pdbCount := 0
147+
for _, action := range actions {
148+
if action.GetResource().Resource == "deployments" {
149+
deploymentCount++
150+
} else if action.GetResource().Resource == "services" {
151+
serviceCount++
152+
} else if action.GetResource().Resource == "poddisruptionbudgets" {
153+
pdbCount++
154+
}
155+
}
156+
157+
if deploymentCount != 2 {
158+
t.Errorf("expected 2 deployment actions (create + get), but got %d", deploymentCount)
159+
}
160+
161+
if serviceCount != 1 {
162+
t.Errorf("expected 1 service action, but got %d", serviceCount)
163+
}
164+
165+
if pdbCount != 1 {
166+
t.Errorf("expected 1 PDB action, but got %d", pdbCount)
113167
}
114168
}
115169

@@ -152,9 +206,14 @@ func TestInstallKarmadaAPIServer(t *testing.T) {
152206
t.Fatalf("expected no error, but got: %v", err)
153207
}
154208

155-
deployment, err := verifyDeploymentCreation(fakeClient, &replicas, imagePullPolicy, cfg.ExtraArgs, name, namespace, image, util.KarmadaAPIServerName(name), priorityClassName)
209+
deployment, err := verifyDeploymentCreation(fakeClient)
156210
if err != nil {
157-
t.Fatalf("failed to verify karmada apiserver correct deployment creation correct details: %v", err)
211+
t.Fatalf("failed to verify karmada apiserver deployment creation: %v", err)
212+
}
213+
214+
// Verify deployment details using the existing function
215+
if err := verifyDeploymentDetails(deployment, &replicas, imagePullPolicy, cfg.ExtraArgs, name, namespace, image, util.KarmadaAPIServerName(name), priorityClassName); err != nil {
216+
t.Fatalf("failed to verify deployment details: %v", err)
158217
}
159218

160219
err = verifyAPIServerDeploymentAdditionalDetails(deployment, name, serviceSubnet)
@@ -248,9 +307,9 @@ func TestInstallKarmadaAggregatedAPIServer(t *testing.T) {
248307
t.Fatalf("Failed to install Karmada Aggregated API Server: %v", err)
249308
}
250309

251-
deployment, err := verifyDeploymentCreation(fakeClient, &replicas, imagePullPolicy, cfg.ExtraArgs, name, namespace, image, util.KarmadaAggregatedAPIServerName(name), priorityClassName)
310+
deployment, err := verifyDeploymentCreation(fakeClient)
252311
if err != nil {
253-
t.Fatalf("failed to verify karmada aggregated apiserver deployment creation correct details: %v", err)
312+
t.Fatalf("failed to verify karmada aggregated apiserver deployment creation: %v", err)
254313
}
255314

256315
err = verifyAggregatedAPIServerDeploymentAdditionalDetails(featureGates, deployment, name)
@@ -315,28 +374,33 @@ func contains(slice []string, item string) bool {
315374
// based on the given parameters. It ensures that the deployment has the correct
316375
// number of replicas, image pull policy, extra arguments, and labels, as well
317376
// as the correct image for the Karmada API server.
318-
func verifyDeploymentCreation(client *fakeclientset.Clientset, replicas *int32, imagePullPolicy corev1.PullPolicy, extraArgs map[string]string, name, namespace, image, expectedDeploymentName, priorityClassName string) (*appsv1.Deployment, error) {
319-
// Assert that a Deployment was created.
377+
func verifyDeploymentCreation(client *fakeclientset.Clientset) (*appsv1.Deployment, error) {
378+
// Assert that a Deployment and PDB were created.
320379
actions := client.Actions()
321-
if len(actions) != 1 {
322-
return nil, fmt.Errorf("expected exactly 1 action either create or update, but got %d actions", len(actions))
323-
}
324-
325-
// Check that the action was a Deployment creation.
326-
createAction, ok := actions[0].(coretesting.CreateAction)
327-
if !ok {
328-
return nil, fmt.Errorf("expected a CreateAction, but got %T", actions[0])
329-
}
330-
331-
// Check that the action was performed on the correct resource.
332-
if createAction.GetResource().Resource != "deployments" {
333-
return nil, fmt.Errorf("expected action on 'deployments', but got '%s'", createAction.GetResource().Resource)
380+
// We now create deployment, PDB, and perform a get action, so expect 3 actions
381+
if len(actions) != 3 {
382+
return nil, fmt.Errorf("expected exactly 3 actions (deployment + PDB + get), but got %d actions", len(actions))
383+
}
384+
385+
// Find the deployment action
386+
var deployment *appsv1.Deployment
387+
for _, action := range actions {
388+
if action.GetResource().Resource == "deployments" {
389+
createAction, isCreate := action.(coretesting.CreateAction)
390+
_, isGet := action.(coretesting.GetAction)
391+
if !isCreate && !isGet {
392+
return nil, fmt.Errorf("expected a GetAction or CreateAction for deployment, but got %T", action)
393+
}
394+
if isGet {
395+
continue
396+
}
397+
deployment = createAction.GetObject().(*appsv1.Deployment)
398+
break
399+
}
334400
}
335401

336-
deployment := createAction.GetObject().(*appsv1.Deployment)
337-
err := verifyDeploymentDetails(deployment, replicas, imagePullPolicy, extraArgs, name, namespace, image, expectedDeploymentName, priorityClassName)
338-
if err != nil {
339-
return nil, err
402+
if deployment == nil {
403+
return nil, fmt.Errorf("expected deployment action, but none found")
340404
}
341405

342406
return deployment, nil

operator/pkg/controlplane/controlplane.go

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,20 @@ limitations under the License.
1717
package controlplane
1818

1919
import (
20+
"context"
2021
"fmt"
2122

2223
appsv1 "k8s.io/api/apps/v1"
24+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2325
kuberuntime "k8s.io/apimachinery/pkg/runtime"
26+
"k8s.io/apimachinery/pkg/runtime/schema"
2427
clientset "k8s.io/client-go/kubernetes"
2528
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
2629
"k8s.io/klog/v2"
2730

2831
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
2932
"github.com/karmada-io/karmada/operator/pkg/constants"
33+
"github.com/karmada-io/karmada/operator/pkg/controlplane/pdb"
3034
"github.com/karmada-io/karmada/operator/pkg/util"
3135
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
3236
"github.com/karmada-io/karmada/operator/pkg/util/patcher"
@@ -48,6 +52,50 @@ func EnsureControlPlaneComponent(component, name, namespace string, featureGates
4852
if err := apiclient.CreateOrUpdateDeployment(client, deployment); err != nil {
4953
return fmt.Errorf("failed to create deployment resource for component %s, err: %w", component, err)
5054
}
55+
56+
// Ensure PDB for the component if configured
57+
// Map PascalCase component constants to naming functions for PDB
58+
var (
59+
commonSettings *operatorv1alpha1.CommonSettings
60+
pdbName string
61+
)
62+
switch component {
63+
case constants.KarmadaControllerManagerComponent:
64+
if cfg.KarmadaControllerManager != nil {
65+
commonSettings = &cfg.KarmadaControllerManager.CommonSettings
66+
pdbName = util.KarmadaControllerManagerName(name)
67+
}
68+
case constants.KubeControllerManagerComponent:
69+
if cfg.KubeControllerManager != nil {
70+
commonSettings = &cfg.KubeControllerManager.CommonSettings
71+
pdbName = util.KubeControllerManagerName(name)
72+
}
73+
case constants.KarmadaSchedulerComponent:
74+
if cfg.KarmadaScheduler != nil {
75+
commonSettings = &cfg.KarmadaScheduler.CommonSettings
76+
pdbName = util.KarmadaSchedulerName(name)
77+
}
78+
case constants.KarmadaDeschedulerComponent:
79+
if cfg.KarmadaDescheduler != nil {
80+
commonSettings = &cfg.KarmadaDescheduler.CommonSettings
81+
pdbName = util.KarmadaDeschedulerName(name)
82+
}
83+
}
84+
85+
if commonSettings != nil {
86+
// Fetch the persisted Deployment to ensure UID is populated
87+
persisted, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), deployment.GetName(), metav1.GetOptions{})
88+
if err != nil {
89+
return fmt.Errorf("failed to fetch Deployment %s/%s for PDB owner, err: %w", namespace, deployment.GetName(), err)
90+
}
91+
// Build OwnerReference for controller with real UID
92+
gvk := schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}
93+
ownerRef := *metav1.NewControllerRef(persisted, gvk)
94+
if err := pdb.EnsurePodDisruptionBudget(client, pdbName, namespace, commonSettings.PodDisruptionBudgetConfig, deployment.Spec.Template.Labels, []metav1.OwnerReference{ownerRef}); err != nil {
95+
return fmt.Errorf("failed to ensure PDB for component %s (instance: %s), err: %w", component, pdbName, err)
96+
}
97+
}
98+
5199
return nil
52100
}
53101

operator/pkg/controlplane/controlplane_test.go

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import (
2020
appsv1 "k8s.io/api/apps/v1"
2121
corev1 "k8s.io/api/core/v1"
2222
fakeclientset "k8s.io/client-go/kubernetes/fake"
23-
coretesting "k8s.io/client-go/testing"
2423
"k8s.io/utils/ptr"
2524

2625
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
@@ -111,19 +110,31 @@ func TestEnsureAllControlPlaneComponents(t *testing.T) {
111110
}
112111

113112
actions := fakeClient.Actions()
114-
if len(actions) != len(components) {
115-
t.Fatalf("expected %d actions, but got %d", len(components), len(actions))
113+
// We now create both deployments and PDBs,include get the deployment. so expect 3 actions per component
114+
expectedActions := len(components) * 3
115+
if len(actions) != expectedActions {
116+
t.Fatalf("expected %d actions, but got %d", expectedActions, len(actions))
116117
}
117118

119+
// Check that we have both deployments and PDBs
120+
deploymentCount := 0
121+
pdbCount := 0
118122
for _, action := range actions {
119-
createAction, ok := action.(coretesting.CreateAction)
120-
if !ok {
121-
t.Errorf("expected CreateAction, but got %T", action)
123+
if action.GetResource().Resource == "deployments" {
124+
deploymentCount++
125+
} else if action.GetResource().Resource == "poddisruptionbudgets" {
126+
pdbCount++
122127
}
128+
}
123129

124-
if createAction.GetResource().Resource != "deployments" {
125-
t.Errorf("expected action on 'deployments', but got '%s'", createAction.GetResource().Resource)
126-
}
130+
// Each component has 2 deployment actions (create + get) and 1 PDB action
131+
expectedDeployments := len(components) * 2
132+
if deploymentCount != expectedDeployments {
133+
t.Errorf("expected %d deployment actions (create + get for each component), but got %d", expectedDeployments, deploymentCount)
134+
}
135+
136+
if pdbCount != len(components) {
137+
t.Errorf("expected %d PDB actions, but got %d", len(components), pdbCount)
127138
}
128139
}
129140

operator/pkg/controlplane/etcd/etcd.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,22 @@ limitations under the License.
1717
package etcd
1818

1919
import (
20+
"context"
2021
"fmt"
2122
"strings"
2223

2324
appsv1 "k8s.io/api/apps/v1"
2425
corev1 "k8s.io/api/core/v1"
26+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2527
kuberuntime "k8s.io/apimachinery/pkg/runtime"
28+
"k8s.io/apimachinery/pkg/runtime/schema"
2629
clientset "k8s.io/client-go/kubernetes"
2730
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
2831
"k8s.io/component-base/cli/flag"
2932

3033
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
3134
"github.com/karmada-io/karmada/operator/pkg/constants"
35+
"github.com/karmada-io/karmada/operator/pkg/controlplane/pdb"
3236
"github.com/karmada-io/karmada/operator/pkg/util"
3337
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
3438
"github.com/karmada-io/karmada/operator/pkg/util/patcher"
@@ -98,6 +102,17 @@ func installKarmadaEtcd(client clientset.Interface, name, namespace string, cfg
98102
return fmt.Errorf("error when creating Etcd statefulset, err: %w", err)
99103
}
100104

105+
// Fetch persisted StatefulSet to get real UID
106+
persisted, err := client.AppsV1().StatefulSets(namespace).Get(context.TODO(), etcdStatefulSet.GetName(), metav1.GetOptions{})
107+
if err != nil {
108+
return fmt.Errorf("failed to fetch StatefulSet %s/%s for PDB owner, err: %w", namespace, etcdStatefulSet.GetName(), err)
109+
}
110+
gvk := schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"}
111+
ownerRef := *metav1.NewControllerRef(persisted, gvk)
112+
if err := pdb.EnsurePodDisruptionBudget(client, util.KarmadaEtcdName(name), namespace, cfg.CommonSettings.PodDisruptionBudgetConfig, etcdStatefulSet.Spec.Template.Labels, []metav1.OwnerReference{ownerRef}); err != nil {
113+
return fmt.Errorf("failed to ensure PDB for etcd component %s, err: %w", util.KarmadaEtcdName(name), err)
114+
}
115+
101116
return nil
102117
}
103118

0 commit comments

Comments
 (0)