Skip to content

Commit 748dbab

Browse files
leon-apeleon-ape
authored
feat: support exclusive replica roles to prevent stale labels (#10049)
1 parent b893218 commit 748dbab

10 files changed

+347
-15
lines changed

‎apis/apps/v1/componentdefinition_types.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1456,6 +1456,18 @@ type ReplicaRole struct {
14561456
// +kubebuilder:default=false
14571457
// +optional
14581458
ParticipatesInQuorum bool `json:"participatesInQuorum"`
1459+
1460+
// IsExclusive specifies if this role can be assigned to only one Pod at a time
1461+
// within a Component. If true, the controller ensures that when a new Pod
1462+
// claims this role, any existing Pods with the same role label will have
1463+
// their labels removed immediately.
1464+
// This helps prevent "Split-Brain" scenarios during network partitions or node failures.
1465+
//
1466+
// This field is immutable once set.
1467+
//
1468+
// +kubebuilder:default=false
1469+
// +optional
1470+
IsExclusive bool `json:"isExclusive"`
14591471
}
14601472

14611473
// UpdateStrategy defines the update strategy for cluster components. This strategy determines how updates are applied

‎config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10965,6 +10965,18 @@ spec:
1096510965
description: ReplicaRole represents a role that can be assigned
1096610966
to a component instance, defining its behavior and responsibilities.
1096710967
properties:
10968+
isExclusive:
10969+
default: false
10970+
description: |-
10971+
IsExclusive specifies if this role can be assigned to only one Pod at a time
10972+
within a Component. If true, the controller ensures that when a new Pod
10973+
claims this role, any existing Pods with the same role label will have
10974+
their labels removed immediately.
10975+
This helps prevent "Split-Brain" scenarios during network partitions or node failures.
10976+
10977+
10978+
This field is immutable once set.
10979+
type: boolean
1096810980
name:
1096910981
description: |-
1097010982
Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label

‎config/crd/bases/workloads.kubeblocks.io_instances.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2017,6 +2017,18 @@ spec:
20172017
description: ReplicaRole represents a role that can be assigned
20182018
to a component instance, defining its behavior and responsibilities.
20192019
properties:
2020+
isExclusive:
2021+
default: false
2022+
description: |-
2023+
IsExclusive specifies if this role can be assigned to only one Pod at a time
2024+
within a Component. If true, the controller ensures that when a new Pod
2025+
claims this role, any existing Pods with the same role label will have
2026+
their labels removed immediately.
2027+
This helps prevent "Split-Brain" scenarios during network partitions or node failures.
2028+
2029+
2030+
This field is immutable once set.
2031+
type: boolean
20202032
name:
20212033
description: |-
20222034
Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label

‎config/crd/bases/workloads.kubeblocks.io_instancesets.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3557,6 +3557,18 @@ spec:
35573557
description: ReplicaRole represents a role that can be assigned
35583558
to a component instance, defining its behavior and responsibilities.
35593559
properties:
3560+
isExclusive:
3561+
default: false
3562+
description: |-
3563+
IsExclusive specifies if this role can be assigned to only one Pod at a time
3564+
within a Component. If true, the controller ensures that when a new Pod
3565+
claims this role, any existing Pods with the same role label will have
3566+
their labels removed immediately.
3567+
This helps prevent "Split-Brain" scenarios during network partitions or node failures.
3568+
3569+
3570+
This field is immutable once set.
3571+
type: boolean
35603572
name:
35613573
description: |-
35623574
Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label

‎deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10965,6 +10965,18 @@ spec:
1096510965
description: ReplicaRole represents a role that can be assigned
1096610966
to a component instance, defining its behavior and responsibilities.
1096710967
properties:
10968+
isExclusive:
10969+
default: false
10970+
description: |-
10971+
IsExclusive specifies if this role can be assigned to only one Pod at a time
10972+
within a Component. If true, the controller ensures that when a new Pod
10973+
claims this role, any existing Pods with the same role label will have
10974+
their labels removed immediately.
10975+
This helps prevent "Split-Brain" scenarios during network partitions or node failures.
10976+
10977+
10978+
This field is immutable once set.
10979+
type: boolean
1096810980
name:
1096910981
description: |-
1097010982
Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label

‎deploy/helm/crds/workloads.kubeblocks.io_instances.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2017,6 +2017,18 @@ spec:
20172017
description: ReplicaRole represents a role that can be assigned
20182018
to a component instance, defining its behavior and responsibilities.
20192019
properties:
2020+
isExclusive:
2021+
default: false
2022+
description: |-
2023+
IsExclusive specifies if this role can be assigned to only one Pod at a time
2024+
within a Component. If true, the controller ensures that when a new Pod
2025+
claims this role, any existing Pods with the same role label will have
2026+
their labels removed immediately.
2027+
This helps prevent "Split-Brain" scenarios during network partitions or node failures.
2028+
2029+
2030+
This field is immutable once set.
2031+
type: boolean
20202032
name:
20212033
description: |-
20222034
Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label

‎deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3557,6 +3557,18 @@ spec:
35573557
description: ReplicaRole represents a role that can be assigned
35583558
to a component instance, defining its behavior and responsibilities.
35593559
properties:
3560+
isExclusive:
3561+
default: false
3562+
description: |-
3563+
IsExclusive specifies if this role can be assigned to only one Pod at a time
3564+
within a Component. If true, the controller ensures that when a new Pod
3565+
claims this role, any existing Pods with the same role label will have
3566+
their labels removed immediately.
3567+
This helps prevent "Split-Brain" scenarios during network partitions or node failures.
3568+
3569+
3570+
This field is immutable once set.
3571+
type: boolean
35603572
name:
35613573
description: |-
35623574
Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label

‎docs/developer_docs/api-reference/cluster.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10374,6 +10374,23 @@ of 2 learners and 1 follower while maintaining quorum.</p>
1037410374
<p>This field is immutable once set.</p>
1037510375
</td>
1037610376
</tr>
10377+
<tr>
10378+
<td>
10379+
<code>isExclusive</code><br/>
10380+
<em>
10381+
bool
10382+
</em>
10383+
</td>
10384+
<td>
10385+
<em>(Optional)</em>
10386+
<p>IsExclusive specifies if this role can be assigned to only one Pod at a time
10387+
within a Component. If true, the controller ensures that when a new Pod
10388+
claims this role, any existing Pods with the same role label will have
10389+
their labels removed immediately.
10390+
This helps prevent &ldquo;Split-Brain&rdquo; scenarios during network partitions or node failures.</p>
10391+
<p>This field is immutable once set.</p>
10392+
</td>
10393+
</tr>
1037710394
</tbody>
1037810395
</table>
1037910396
<h3 id="apps.kubeblocks.io/v1.ReplicasLimit">ReplicasLimit

‎pkg/controller/instanceset/pod_role_event_handler.go‎

Lines changed: 44 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ package instanceset
2121

2222
import (
2323
"encoding/json"
24+
"errors"
2425
"fmt"
2526
"regexp"
2627
"slices"
@@ -236,27 +237,55 @@ func parseProbeEventMessage(reqCtx intctrlutil.RequestCtx, event *corev1.Event)
236237
return nil
237238
}
238239

239-
// updatePodRoleLabel updates pod role label when internal container role changed
240-
func updatePodRoleLabel(cli client.Client, reqCtx intctrlutil.RequestCtx,
241-
its workloads.InstanceSet, pod *corev1.Pod, roleName string, version string) error {
242-
ctx := reqCtx.Ctx
243-
roleMap := composeRoleMap(its)
244-
// role not defined in CR, ignore it
245-
roleName = strings.ToLower(roleName)
246-
240+
func updatePodRoleLabel(cli client.Client, reqCtx intctrlutil.RequestCtx, its workloads.InstanceSet,
241+
pod *corev1.Pod, roleName string, version string) error {
242+
var (
243+
ctx = reqCtx.Ctx
244+
roleMap = composeRoleMap(its)
245+
normalizedRoleName = strings.ToLower(roleName)
246+
role, defined = roleMap[normalizedRoleName]
247+
)
247248
// update pod role label
248249
newPod := pod.DeepCopy()
249-
role, ok := roleMap[roleName]
250-
switch ok {
251-
case true:
252-
newPod.Labels[RoleLabelKey] = role.Name
253-
case false:
250+
if defined {
251+
newPod.Labels[RoleLabelKey] = normalizedRoleName
252+
} else {
254253
delete(newPod.Labels, RoleLabelKey)
255254
}
256-
257255
if newPod.Annotations == nil {
258256
newPod.Annotations = map[string]string{}
259257
}
260258
newPod.Annotations[constant.LastRoleSnapshotVersionAnnotationKey] = version
261-
return cli.Update(ctx, newPod)
259+
if err := cli.Update(ctx, newPod); err != nil {
260+
return err
261+
}
262+
263+
if role.IsExclusive {
264+
return removeExclusiveRoleLabels(cli, reqCtx, its, pod.Name, normalizedRoleName)
265+
}
266+
return nil
267+
}
268+
269+
func removeExclusiveRoleLabels(cli client.Client, reqCtx intctrlutil.RequestCtx, its workloads.InstanceSet, newPodName, roleName string) error {
270+
labels := getMatchLabels(its.Name)
271+
labels[RoleLabelKey] = roleName
272+
var pods corev1.PodList
273+
if err := cli.List(reqCtx.Ctx, &pods, client.InNamespace(its.Namespace), client.MatchingLabels(labels)); err != nil {
274+
return err
275+
}
276+
277+
var errs []error
278+
for i, pod := range pods.Items {
279+
if pod.Name == newPodName {
280+
continue
281+
}
282+
newPod := pods.Items[i].DeepCopy()
283+
delete(newPod.Labels, RoleLabelKey)
284+
if err := cli.Update(reqCtx.Ctx, newPod); err != nil {
285+
errs = append(errs, err)
286+
} else {
287+
reqCtx.Log.Info("remove exclusive role label", "pod", newPod.Name, "role", roleName)
288+
}
289+
}
290+
return errors.Join(errs...)
262291
}

0 commit comments

Comments
 (0)