Skip to content

Commit 608471a

Browse files
zustonJunfan Zhang
andauthored
[#2362] feat(client): Log more compression infos (#2367)
### What changes were proposed in this pull request? Followup to log more compression infos like data len and compression ratio ### Why are the changes needed? followup #2362 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Needn't --------- Co-authored-by: Junfan Zhang <zhangjunfan@qiyi.com>
1 parent 4cb8c69 commit 608471a

File tree

1 file changed

+8
-2
lines changed

1 file changed

+8
-2
lines changed

client-spark/common/src/main/java/org/apache/spark/shuffle/writer/WriteBufferManager.java

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ public class WriteBufferManager extends MemoryConsumer {
9292
private SerializationStream serializeStream;
9393
private WrappedByteArrayOutputStream arrayOutputStream;
9494
private long uncompressedDataLen = 0;
95+
private long compressedDataLen = 0;
9596
private long requireMemoryInterval;
9697
private int requireMemoryRetryMax;
9798
private Optional<Codec> codec;
@@ -430,6 +431,7 @@ protected ShuffleBlockInfo createShuffleBlock(int partitionId, WriterBuffer wb)
430431
blockIdLayout.getBlockId(getNextSeqNo(partitionId), partitionId, taskAttemptId);
431432
blockCounter.incrementAndGet();
432433
uncompressedDataLen += data.length;
434+
compressedDataLen += compressed.length;
433435
shuffleWriteMetrics.incBytesWritten(compressed.length);
434436
// add memory to indicate bytes which will be sent to shuffle server
435437
inSendListBytes.addAndGet(wb.getMemoryUsed());
@@ -648,14 +650,18 @@ public String getManagerCostInfo() {
648650
+ serializeTime
649651
+ "], sortTime["
650652
+ sortTime
651-
+ "], compressTime["
652-
+ compressTime
653653
+ "], estimateTime["
654654
+ estimateTime
655655
+ "], requireMemoryTime["
656656
+ requireMemoryTime
657657
+ "], uncompressedDataLen["
658658
+ uncompressedDataLen
659+
+ "], compressedDataLen["
660+
+ compressedDataLen
661+
+ "], compressTime["
662+
+ compressTime
663+
+ "], compressRatio["
664+
+ (compressedDataLen == 0 ? 0 : (float) uncompressedDataLen / compressedDataLen)
659665
+ "]";
660666
}
661667

0 commit comments

Comments
 (0)