@@ -2249,28 +2249,38 @@ private CoordinatorResult<ConsumerGroupHeartbeatResponseData, CoordinatorRecord>
2249
2249
.setClassicMemberMetadata (null )
2250
2250
.build ();
2251
2251
2252
- // If the group is newly created, we must ensure that it moves away from
2253
- // epoch 0 and that it is fully initialized.
2254
- boolean bumpGroupEpoch = group .groupEpoch () == 0 ;
2255
-
2256
- bumpGroupEpoch |= hasMemberSubscriptionChanged (
2252
+ boolean subscribedTopicNamesChanged = hasMemberSubscriptionChanged (
2257
2253
groupId ,
2258
2254
member ,
2259
2255
updatedMember ,
2260
2256
records
2261
2257
);
2262
-
2263
- bumpGroupEpoch |= maybeUpdateRegularExpressions (
2258
+ UpdateRegularExpressionsResult updateRegularExpressionsResult = maybeUpdateRegularExpressions (
2264
2259
context ,
2265
2260
group ,
2266
2261
member ,
2267
2262
updatedMember ,
2268
2263
records
2269
2264
);
2270
2265
2266
+ // The subscription has changed when either the subscribed topic names or subscribed topic
2267
+ // regex has changed.
2268
+ boolean hasSubscriptionChanged = subscribedTopicNamesChanged || updateRegularExpressionsResult .regexUpdated ();
2271
2269
int groupEpoch = group .groupEpoch ();
2272
2270
SubscriptionType subscriptionType = group .subscriptionType ();
2273
2271
2272
+ boolean bumpGroupEpoch =
2273
+ // If the group is newly created, we must ensure that it moves away from
2274
+ // epoch 0 and that it is fully initialized.
2275
+ groupEpoch == 0 ||
2276
+ // Bumping the group epoch signals that the target assignment should be updated. We bump
2277
+ // the group epoch when the member has changed its subscribed topic names or the member
2278
+ // has changed its subscribed topic regex to a regex that is already resolved. We avoid
2279
+ // bumping the group epoch when the new subscribed topic regex has not been resolved
2280
+ // yet, since we will have to update the target assignment again later.
2281
+ subscribedTopicNamesChanged ||
2282
+ updateRegularExpressionsResult == UpdateRegularExpressionsResult .REGEX_UPDATED_AND_RESOLVED ;
2283
+
2274
2284
if (bumpGroupEpoch || group .hasMetadataExpired (currentTimeMs )) {
2275
2285
// The subscription metadata is updated in two cases:
2276
2286
// 1) The member has updated its subscriptions;
@@ -2315,6 +2325,9 @@ private CoordinatorResult<ConsumerGroupHeartbeatResponseData, CoordinatorRecord>
2315
2325
group ::currentPartitionEpoch ,
2316
2326
targetAssignmentEpoch ,
2317
2327
targetAssignment ,
2328
+ group .resolvedRegularExpressions (),
2329
+ // Force consistency with the subscription when the subscription has changed.
2330
+ hasSubscriptionChanged ,
2318
2331
ownedTopicPartitions ,
2319
2332
records
2320
2333
);
@@ -2468,6 +2481,8 @@ private CoordinatorResult<Void, CoordinatorRecord> classicGroupJoinToConsumerGro
2468
2481
group ::currentPartitionEpoch ,
2469
2482
group .assignmentEpoch (),
2470
2483
group .targetAssignment (updatedMember .memberId (), updatedMember .instanceId ()),
2484
+ group .resolvedRegularExpressions (),
2485
+ bumpGroupEpoch ,
2471
2486
toTopicPartitions (subscription .ownedPartitions (), metadataImage ),
2472
2487
records
2473
2488
);
@@ -2511,6 +2526,9 @@ private CoordinatorResult<Void, CoordinatorRecord> classicGroupJoinToConsumerGro
2511
2526
group ::currentPartitionEpoch ,
2512
2527
targetAssignmentEpoch ,
2513
2528
targetAssignment ,
2529
+ group .resolvedRegularExpressions (),
2530
+ // Force consistency with the subscription when the subscription has changed.
2531
+ bumpGroupEpoch ,
2514
2532
toTopicPartitions (subscription .ownedPartitions (), metadataImage ),
2515
2533
records
2516
2534
);
@@ -2669,6 +2687,8 @@ private CoordinatorResult<Map.Entry<ShareGroupHeartbeatResponseData, Optional<In
2669
2687
updatedMember ,
2670
2688
targetAssignmentEpoch ,
2671
2689
targetAssignment ,
2690
+ // Force consistency with the subscription when the subscription has changed.
2691
+ bumpGroupEpoch ,
2672
2692
records
2673
2693
);
2674
2694
@@ -3108,6 +3128,16 @@ private static boolean isNotEmpty(String value) {
3108
3128
return value != null && !value .isEmpty ();
3109
3129
}
3110
3130
3131
+ private enum UpdateRegularExpressionsResult {
3132
+ NO_CHANGE ,
3133
+ REGEX_UPDATED ,
3134
+ REGEX_UPDATED_AND_RESOLVED ;
3135
+
3136
+ public boolean regexUpdated () {
3137
+ return this == REGEX_UPDATED || this == REGEX_UPDATED_AND_RESOLVED ;
3138
+ }
3139
+ }
3140
+
3111
3141
/**
3112
3142
* Check whether the member has updated its subscribed topic regular expression and
3113
3143
* may trigger the resolution/the refresh of all the regular expressions in the
@@ -3119,9 +3149,9 @@ private static boolean isNotEmpty(String value) {
3119
3149
* @param member The old member.
3120
3150
* @param updatedMember The new member.
3121
3151
* @param records The records accumulator.
3122
- * @return Whether a rebalance must be triggered .
3152
+ * @return The result of the update .
3123
3153
*/
3124
- private boolean maybeUpdateRegularExpressions (
3154
+ private UpdateRegularExpressionsResult maybeUpdateRegularExpressions (
3125
3155
AuthorizableRequestContext context ,
3126
3156
ConsumerGroup group ,
3127
3157
ConsumerGroupMember member ,
@@ -3134,14 +3164,17 @@ private boolean maybeUpdateRegularExpressions(
3134
3164
String oldSubscribedTopicRegex = member .subscribedTopicRegex ();
3135
3165
String newSubscribedTopicRegex = updatedMember .subscribedTopicRegex ();
3136
3166
3137
- boolean bumpGroupEpoch = false ;
3138
3167
boolean requireRefresh = false ;
3168
+ UpdateRegularExpressionsResult updateRegularExpressionsResult = UpdateRegularExpressionsResult .NO_CHANGE ;
3139
3169
3140
3170
// Check whether the member has changed its subscribed regex.
3141
- if (!Objects .equals (oldSubscribedTopicRegex , newSubscribedTopicRegex )) {
3171
+ boolean subscribedTopicRegexChanged = !Objects .equals (oldSubscribedTopicRegex , newSubscribedTopicRegex );
3172
+ if (subscribedTopicRegexChanged ) {
3142
3173
log .debug ("[GroupId {}] Member {} updated its subscribed regex to: {}." ,
3143
3174
groupId , memberId , newSubscribedTopicRegex );
3144
3175
3176
+ updateRegularExpressionsResult = UpdateRegularExpressionsResult .REGEX_UPDATED ;
3177
+
3145
3178
if (isNotEmpty (oldSubscribedTopicRegex ) && group .numSubscribedMembers (oldSubscribedTopicRegex ) == 1 ) {
3146
3179
// If the member was the last one subscribed to the regex, we delete the
3147
3180
// resolved regular expression.
@@ -3160,7 +3193,9 @@ private boolean maybeUpdateRegularExpressions(
3160
3193
} else {
3161
3194
// If the new regex is already resolved, we trigger a rebalance
3162
3195
// by bumping the group epoch.
3163
- bumpGroupEpoch = group .resolvedRegularExpression (newSubscribedTopicRegex ).isPresent ();
3196
+ if (group .resolvedRegularExpression (newSubscribedTopicRegex ).isPresent ()) {
3197
+ updateRegularExpressionsResult = UpdateRegularExpressionsResult .REGEX_UPDATED_AND_RESOLVED ;
3198
+ }
3164
3199
}
3165
3200
}
3166
3201
}
@@ -3176,20 +3211,20 @@ private boolean maybeUpdateRegularExpressions(
3176
3211
// 0. The group is subscribed to regular expressions. We also take the one
3177
3212
// that the current may have just introduced.
3178
3213
if (!requireRefresh && group .subscribedRegularExpressions ().isEmpty ()) {
3179
- return bumpGroupEpoch ;
3214
+ return updateRegularExpressionsResult ;
3180
3215
}
3181
3216
3182
3217
// 1. There is no ongoing refresh for the group.
3183
3218
String key = group .groupId () + "-regex" ;
3184
3219
if (executor .isScheduled (key )) {
3185
- return bumpGroupEpoch ;
3220
+ return updateRegularExpressionsResult ;
3186
3221
}
3187
3222
3188
3223
// 2. The last refresh is older than 10s. If the group does not have any regular
3189
3224
// expressions but the current member just brought a new one, we should continue.
3190
3225
long lastRefreshTimeMs = group .lastResolvedRegularExpressionRefreshTimeMs ();
3191
3226
if (currentTimeMs <= lastRefreshTimeMs + REGEX_BATCH_REFRESH_MIN_INTERVAL_MS ) {
3192
- return bumpGroupEpoch ;
3227
+ return updateRegularExpressionsResult ;
3193
3228
}
3194
3229
3195
3230
// 3.1 The group has unresolved regular expressions.
@@ -3218,7 +3253,7 @@ private boolean maybeUpdateRegularExpressions(
3218
3253
);
3219
3254
}
3220
3255
3221
- return bumpGroupEpoch ;
3256
+ return updateRegularExpressionsResult ;
3222
3257
}
3223
3258
3224
3259
/**
@@ -3492,16 +3527,18 @@ private boolean hasStreamsMemberMetadataChanged(
3492
3527
/**
3493
3528
* Reconciles the current assignment of the member towards the target assignment if needed.
3494
3529
*
3495
- * @param groupId The group id.
3496
- * @param member The member to reconcile.
3497
- * @param currentPartitionEpoch The function returning the current epoch of
3498
- * a given partition.
3499
- * @param targetAssignmentEpoch The target assignment epoch.
3500
- * @param targetAssignment The target assignment.
3501
- * @param ownedTopicPartitions The list of partitions owned by the member. This
3502
- * is reported in the ConsumerGroupHeartbeat API and
3503
- * it could be null if not provided.
3504
- * @param records The list to accumulate any new records.
3530
+ * @param groupId The group id.
3531
+ * @param member The member to reconcile.
3532
+ * @param currentPartitionEpoch The function returning the current epoch of
3533
+ * a given partition.
3534
+ * @param targetAssignmentEpoch The target assignment epoch.
3535
+ * @param targetAssignment The target assignment.
3536
+ * @param resolvedRegularExpressions The resolved regular expressions.
3537
+ * @param hasSubscriptionChanged Whether the member has changed its subscription on the current heartbeat.
3538
+ * @param ownedTopicPartitions The list of partitions owned by the member. This
3539
+ * is reported in the ConsumerGroupHeartbeat API and
3540
+ * it could be null if not provided.
3541
+ * @param records The list to accumulate any new records.
3505
3542
* @return The received member if no changes have been made; or a new
3506
3543
* member containing the new assignment.
3507
3544
*/
@@ -3511,15 +3548,20 @@ private ConsumerGroupMember maybeReconcile(
3511
3548
BiFunction <Uuid , Integer , Integer > currentPartitionEpoch ,
3512
3549
int targetAssignmentEpoch ,
3513
3550
Assignment targetAssignment ,
3551
+ Map <String , ResolvedRegularExpression > resolvedRegularExpressions ,
3552
+ boolean hasSubscriptionChanged ,
3514
3553
List <ConsumerGroupHeartbeatRequestData .TopicPartitions > ownedTopicPartitions ,
3515
3554
List <CoordinatorRecord > records
3516
3555
) {
3517
- if (member .isReconciledTo (targetAssignmentEpoch )) {
3556
+ if (! hasSubscriptionChanged && member .isReconciledTo (targetAssignmentEpoch )) {
3518
3557
return member ;
3519
3558
}
3520
3559
3521
3560
ConsumerGroupMember updatedMember = new CurrentAssignmentBuilder (member )
3561
+ .withMetadataImage (metadataImage )
3522
3562
.withTargetAssignment (targetAssignmentEpoch , targetAssignment )
3563
+ .withHasSubscriptionChanged (hasSubscriptionChanged )
3564
+ .withResolvedRegularExpressions (resolvedRegularExpressions )
3523
3565
.withCurrentPartitionEpoch (currentPartitionEpoch )
3524
3566
.withOwnedTopicPartitions (ownedTopicPartitions )
3525
3567
.build ();
@@ -3556,11 +3598,12 @@ private ConsumerGroupMember maybeReconcile(
3556
3598
/**
3557
3599
* Reconciles the current assignment of the member towards the target assignment if needed.
3558
3600
*
3559
- * @param groupId The group id.
3560
- * @param member The member to reconcile.
3561
- * @param targetAssignmentEpoch The target assignment epoch.
3562
- * @param targetAssignment The target assignment.
3563
- * @param records The list to accumulate any new records.
3601
+ * @param groupId The group id.
3602
+ * @param member The member to reconcile.
3603
+ * @param targetAssignmentEpoch The target assignment epoch.
3604
+ * @param targetAssignment The target assignment.
3605
+ * @param hasSubscriptionChanged Whether the member has changed its subscription on the current heartbeat.
3606
+ * @param records The list to accumulate any new records.
3564
3607
* @return The received member if no changes have been made; or a new
3565
3608
* member containing the new assignment.
3566
3609
*/
@@ -3569,14 +3612,17 @@ private ShareGroupMember maybeReconcile(
3569
3612
ShareGroupMember member ,
3570
3613
int targetAssignmentEpoch ,
3571
3614
Assignment targetAssignment ,
3615
+ boolean hasSubscriptionChanged ,
3572
3616
List <CoordinatorRecord > records
3573
3617
) {
3574
- if (member .isReconciledTo (targetAssignmentEpoch )) {
3618
+ if (! hasSubscriptionChanged && member .isReconciledTo (targetAssignmentEpoch )) {
3575
3619
return member ;
3576
3620
}
3577
3621
3578
3622
ShareGroupMember updatedMember = new ShareGroupAssignmentBuilder (member )
3623
+ .withMetadataImage (metadataImage )
3579
3624
.withTargetAssignment (targetAssignmentEpoch , targetAssignment )
3625
+ .withHasSubscriptionChanged (hasSubscriptionChanged )
3580
3626
.build ();
3581
3627
3582
3628
if (!updatedMember .equals (member )) {
0 commit comments