Skip to content

Commit a2db99b

Browse files
zustonJunfan Zhang
andcommitted
improvement(spark): Record the partition choosing sort time for analysis (apache#2364)
### What changes were proposed in this pull request? Record the partition choosing sort time for analysis ### Why are the changes needed? For better optimize write duration ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Needn't Co-authored-by: Junfan Zhang <zhangjunfan@qiyi.com>
1 parent 0558399 commit a2db99b

File tree

1 file changed

+5
-0
lines changed

1 file changed

+5
-0
lines changed

client-spark/common/src/main/java/org/apache/spark/shuffle/writer/WriteBufferManager.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ public class WriteBufferManager extends MemoryConsumer {
8484
private long copyTime = 0;
8585
private long serializeTime = 0;
8686
private long compressTime = 0;
87+
private long sortTime = 0;
8788
private long writeTime = 0;
8889
private long estimateTime = 0;
8990
private long requireMemoryTime = 0;
@@ -335,9 +336,11 @@ public synchronized List<ShuffleBlockInfo> clear(double bufferSpillRatio) {
335336
bufferSpillRatio = Math.max(0.1, Math.min(1.0, bufferSpillRatio));
336337
List<Integer> partitionList = new ArrayList(buffers.keySet());
337338
if (Double.compare(bufferSpillRatio, 1.0) < 0) {
339+
long start = System.currentTimeMillis();
338340
partitionList.sort(
339341
Comparator.comparingInt(o -> buffers.get(o) == null ? 0 : buffers.get(o).getMemoryUsed())
340342
.reversed());
343+
sortTime += start;
341344
targetSpillSize = (long) ((getUsedBytes() - getInSendListBytes()) * bufferSpillRatio);
342345
}
343346

@@ -607,6 +610,8 @@ public String getManagerCostInfo() {
607610
+ writeTime
608611
+ "], serializeTime["
609612
+ serializeTime
613+
+ "], sortTime["
614+
+ sortTime
610615
+ "], compressTime["
611616
+ compressTime
612617
+ "], estimateTime["

0 commit comments

Comments
 (0)