Skip to content

Commit 69dd44a

Browse files
LantaoJinsquito
authored andcommitted
[SPARK-27216][CORE] Upgrade RoaringBitmap to 0.7.45 to fix Kryo unsafe ser/dser issue
## What changes were proposed in this pull request? HighlyCompressedMapStatus uses RoaringBitmap to record the empty blocks. But RoaringBitmap couldn't be ser/deser with unsafe KryoSerializer. It's a bug of RoaringBitmap-0.5.11 and fixed in latest version. This is an update of apache#24157 ## How was this patch tested? Add a UT Closes apache#24264 from LantaoJin/SPARK-27216. Lead-authored-by: LantaoJin <[email protected]> Co-authored-by: Lantao Jin <[email protected]> Signed-off-by: Imran Rashid <[email protected]>
1 parent b517636 commit 69dd44a

File tree

4 files changed

+14
-28
lines changed

4 files changed

+14
-28
lines changed

core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala

Lines changed: 9 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
package org.apache.spark.serializer
1919

20-
import java.io.{ByteArrayInputStream, ByteArrayOutputStream, FileInputStream, FileOutputStream}
20+
import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
2121
import java.nio.ByteBuffer
2222
import java.util.concurrent.Executors
2323

@@ -370,30 +370,6 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
370370
assert(thrown.getCause.isInstanceOf[KryoException])
371371
}
372372

373-
test("SPARK-12222: deserialize RoaringBitmap throw Buffer underflow exception") {
374-
withTempDir { dir =>
375-
val tmpfile = dir.toString + "/RoaringBitmap"
376-
val outStream = new FileOutputStream(tmpfile)
377-
val output = new KryoOutput(outStream)
378-
val bitmap = new RoaringBitmap
379-
bitmap.add(1)
380-
bitmap.add(3)
381-
bitmap.add(5)
382-
// Ignore Kryo because it doesn't use writeObject
383-
bitmap.serialize(new KryoOutputObjectOutputBridge(null, output))
384-
output.flush()
385-
output.close()
386-
387-
val inStream = new FileInputStream(tmpfile)
388-
val input = new KryoInput(inStream)
389-
val ret = new RoaringBitmap
390-
// Ignore Kryo because it doesn't use readObject
391-
ret.deserialize(new KryoInputObjectInputBridge(null, input))
392-
input.close()
393-
assert(ret == bitmap)
394-
}
395-
}
396-
397373
test("KryoOutputObjectOutputBridge.writeObject and KryoInputObjectInputBridge.readObject") {
398374
val kryo = new KryoSerializer(conf).newKryo()
399375

@@ -518,6 +494,14 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
518494
assert(ThreadUtils.awaitResult(f, 10.seconds))
519495
}
520496
}
497+
498+
test("SPARK-27216: test RoaringBitmap ser/dser with Kryo") {
499+
val expected = new RoaringBitmap()
500+
expected.add(1787)
501+
val ser = new KryoSerializer(conf).newInstance()
502+
val actual: RoaringBitmap = ser.deserialize(ser.serialize(expected))
503+
assert(actual === expected)
504+
}
521505
}
522506

523507
class KryoSerializerAutoResetDisabledSuite extends SparkFunSuite with SharedSparkContext {

dev/deps/spark-deps-hadoop-2.7

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
JavaEWAH-0.3.2.jar
2-
RoaringBitmap-0.5.11.jar
2+
RoaringBitmap-0.7.45.jar
33
ST4-4.0.4.jar
44
activation-1.1.1.jar
55
aircompressor-0.10.jar
@@ -178,6 +178,7 @@ scala-parser-combinators_2.12-1.1.0.jar
178178
scala-reflect-2.12.8.jar
179179
scala-xml_2.12-1.0.6.jar
180180
shapeless_2.12-2.3.2.jar
181+
shims-0.7.45.jar
181182
slf4j-api-1.7.16.jar
182183
slf4j-log4j12-1.7.16.jar
183184
snakeyaml-1.23.jar

dev/deps/spark-deps-hadoop-3.2

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
HikariCP-java7-2.4.12.jar
22
JavaEWAH-0.3.2.jar
3-
RoaringBitmap-0.5.11.jar
3+
RoaringBitmap-0.7.45.jar
44
ST4-4.0.4.jar
55
accessors-smart-1.2.jar
66
activation-1.1.1.jar
@@ -198,6 +198,7 @@ scala-parser-combinators_2.12-1.1.0.jar
198198
scala-reflect-2.12.8.jar
199199
scala-xml_2.12-1.0.6.jar
200200
shapeless_2.12-2.3.2.jar
201+
shims-0.7.45.jar
201202
slf4j-api-1.7.16.jar
202203
slf4j-log4j12-1.7.16.jar
203204
snakeyaml-1.23.jar

pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -606,7 +606,7 @@
606606
<dependency>
607607
<groupId>org.roaringbitmap</groupId>
608608
<artifactId>RoaringBitmap</artifactId>
609-
<version>0.5.11</version>
609+
<version>0.7.45</version>
610610
</dependency>
611611
<dependency>
612612
<groupId>commons-net</groupId>

0 commit comments

Comments
 (0)