|
23 | 23 | import java.util.List; |
24 | 24 | import java.util.Map; |
25 | 25 | import java.util.Random; |
| 26 | +import java.util.Set; |
26 | 27 | import java.util.concurrent.TimeUnit; |
27 | 28 | import java.util.concurrent.atomic.AtomicReference; |
28 | 29 |
|
29 | 30 | import com.google.common.annotations.VisibleForTesting; |
30 | 31 | import com.google.common.collect.Lists; |
| 32 | +import com.google.common.collect.Sets; |
31 | 33 | import com.google.protobuf.ByteString; |
32 | 34 | import com.google.protobuf.UnsafeByteOperations; |
33 | 35 | import io.grpc.StatusRuntimeException; |
@@ -540,7 +542,7 @@ public RssSendShuffleDataResponse sendShuffleData(RssSendShuffleDataRequest requ |
540 | 542 |
|
541 | 543 | boolean isSuccessful = true; |
542 | 544 | AtomicReference<StatusCode> failedStatusCode = new AtomicReference<>(StatusCode.INTERNAL_ERROR); |
543 | | - |
| 545 | + Set<Integer> needSplitPartitionIds = Sets.newHashSet(); |
544 | 546 | // prepare rpc request based on shuffleId -> partitionId -> blocks |
545 | 547 | for (Map.Entry<Integer, Map<Integer, List<ShuffleBlockInfo>>> stb : |
546 | 548 | shuffleIdToBlocks.entrySet()) { |
@@ -583,17 +585,18 @@ public RssSendShuffleDataResponse sendShuffleData(RssSendShuffleDataRequest requ |
583 | 585 | RetryUtils.retryWithCondition( |
584 | 586 | () -> { |
585 | 587 | // TODO(baoloongmao): support partition split follow netty client |
586 | | - long requireId = |
| 588 | + Pair<Long, List<Integer>> allocationResult = |
587 | 589 | requirePreAllocation( |
588 | | - appId, |
589 | | - shuffleId, |
590 | | - partitionIds, |
591 | | - partitionRequireSizes, |
592 | | - allocateSize, |
593 | | - request.getRetryMax() / maxRetryAttempts, |
594 | | - request.getRetryIntervalMax(), |
595 | | - failedStatusCode) |
596 | | - .getLeft(); |
| 590 | + appId, |
| 591 | + shuffleId, |
| 592 | + partitionIds, |
| 593 | + partitionRequireSizes, |
| 594 | + allocateSize, |
| 595 | + request.getRetryMax() / maxRetryAttempts, |
| 596 | + request.getRetryIntervalMax(), |
| 597 | + failedStatusCode); |
| 598 | + long requireId = allocationResult.getLeft(); |
| 599 | + needSplitPartitionIds.addAll(allocationResult.getRight()); |
597 | 600 | if (requireId == FAILED_REQUIRE_ID) { |
598 | 601 | throw new RssException( |
599 | 602 | String.format( |
@@ -661,6 +664,7 @@ public RssSendShuffleDataResponse sendShuffleData(RssSendShuffleDataRequest requ |
661 | 664 | } else { |
662 | 665 | response = new RssSendShuffleDataResponse(failedStatusCode.get()); |
663 | 666 | } |
| 667 | + response.setNeedSplitPartitionIds(needSplitPartitionIds); |
664 | 668 | return response; |
665 | 669 | } |
666 | 670 |
|
|
0 commit comments