Skip to content

Commit b60ee3a

Browse files
dilipbiswalueshin
authored andcommitted
[SPARK-25307][SQL] ArraySort function may return an error in the code generation phase
## What changes were proposed in this pull request? Sorting array of booleans (not nullable) returns a compilation error in the code generation phase. Below is the compilation error : ```SQL java.util.concurrent.ExecutionException: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 51, Column 23: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 51, Column 23: No applicable constructor/method found for actual parameters "boolean[]"; candidates are: "public static void java.util.Arrays.sort(long[])", "public static void java.util.Arrays.sort(long[], int, int)", "public static void java.util.Arrays.sort(byte[], int, int)", "public static void java.util.Arrays.sort(float[])", "public static void java.util.Arrays.sort(float[], int, int)", "public static void java.util.Arrays.sort(char[])", "public static void java.util.Arrays.sort(char[], int, int)", "public static void java.util.Arrays.sort(short[], int, int)", "public static void java.util.Arrays.sort(short[])", "public static void java.util.Arrays.sort(byte[])", "public static void java.util.Arrays.sort(java.lang.Object[], int, int, java.util.Comparator)", "public static void java.util.Arrays.sort(java.lang.Object[], java.util.Comparator)", "public static void java.util.Arrays.sort(int[])", "public static void java.util.Arrays.sort(java.lang.Object[], int, int)", "public static void java.util.Arrays.sort(java.lang.Object[])", "public static void java.util.Arrays.sort(double[])", "public static void java.util.Arrays.sort(double[], int, int)", "public static void java.util.Arrays.sort(int[], int, int)" at com.google.common.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:306) at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:293) at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:116) at com.google.common.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:135) at com.google.common.cache.LocalCache$Segment.getAndRecordStats(LocalCache.java:2410) at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2380) at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342) at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2257) at com.google.common.cache.LocalCache.get(LocalCache.java:4000) at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:4004) at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874) at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:1305) ``` ## How was this patch tested? Added test in collectionExpressionSuite Closes apache#22314 from dilipbiswal/SPARK-25307. Authored-by: Dilip Biswal <[email protected]> Signed-off-by: Takuya UESHIN <[email protected]>
1 parent 8e21696 commit b60ee3a

File tree

2 files changed

+10
-2
lines changed

2 files changed

+10
-2
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1046,8 +1046,9 @@ trait ArraySortLike extends ExpectsInputTypes {
10461046
} else {
10471047
s"int $c = ${ctx.genComp(elementType, s"(($jt) $o1)", s"(($jt) $o2)")};"
10481048
}
1049-
val nonNullPrimitiveAscendingSort =
1050-
if (CodeGenerator.isPrimitiveType(elementType) && !containsNull) {
1049+
val canPerformFastSort =
1050+
CodeGenerator.isPrimitiveType(elementType) && elementType != BooleanType && !containsNull
1051+
val nonNullPrimitiveAscendingSort = if (canPerformFastSort) {
10511052
val javaType = CodeGenerator.javaType(elementType)
10521053
val primitiveTypeName = CodeGenerator.primitiveTypeName(elementType)
10531054
s"""

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,12 +326,19 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
326326
val d2 = new Decimal().set(100)
327327
val a4 = Literal.create(Seq(d2, d1), ArrayType(DecimalType(10, 0)))
328328
val a5 = Literal.create(Seq(null, null), ArrayType(NullType))
329+
val a6 = Literal.create(Seq(true, false, true, false),
330+
ArrayType(BooleanType, containsNull = false))
331+
val a7 = Literal.create(Seq(true, false, true, false), ArrayType(BooleanType))
332+
val a8 = Literal.create(Seq(true, false, true, null, false), ArrayType(BooleanType))
329333

330334
checkEvaluation(new SortArray(a0), Seq(1, 2, 3))
331335
checkEvaluation(new SortArray(a1), Seq[Integer]())
332336
checkEvaluation(new SortArray(a2), Seq("a", "b"))
333337
checkEvaluation(new SortArray(a3), Seq(null, "a", "b"))
334338
checkEvaluation(new SortArray(a4), Seq(d1, d2))
339+
checkEvaluation(new SortArray(a6), Seq(false, false, true, true))
340+
checkEvaluation(new SortArray(a7), Seq(false, false, true, true))
341+
checkEvaluation(new SortArray(a8), Seq(null, false, false, true, true))
335342
checkEvaluation(SortArray(a0, Literal(true)), Seq(1, 2, 3))
336343
checkEvaluation(SortArray(a1, Literal(true)), Seq[Integer]())
337344
checkEvaluation(SortArray(a2, Literal(true)), Seq("a", "b"))

0 commit comments

Comments
 (0)