Skip to content

Commit 8e21696

Browse files
dilipbiswalueshin
authored andcommitted
[SPARK-25308][SQL] ArrayContains function may return a error in the code generation phase.
## What changes were proposed in this pull request? Invoking ArrayContains function with non nullable array type throws the following error in the code generation phase. Below is the error snippet. ```SQL Code generation of array_contains([1,2,3], 1) failed: java.util.concurrent.ExecutionException: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 40, Column 11: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 40, Column 11: Expression "isNull_0" is not an rvalue java.util.concurrent.ExecutionException: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 40, Column 11: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 40, Column 11: Expression "isNull_0" is not an rvalue at com.google.common.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:306) at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:293) at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:116) at com.google.common.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:135) at com.google.common.cache.LocalCache$Segment.getAndRecordStats(LocalCache.java:2410) at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2380) at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342) at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2257) at com.google.common.cache.LocalCache.get(LocalCache.java:4000) at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:4004) at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874) at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:1305) ``` ## How was this patch tested? Added test in CollectionExpressionSuite. Closes apache#22315 from dilipbiswal/SPARK-25308. Authored-by: Dilip Biswal <[email protected]> Signed-off-by: Takuya UESHIN <[email protected]>
1 parent 546683c commit 8e21696

File tree

2 files changed

+25
-10
lines changed

2 files changed

+25
-10
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1464,17 +1464,29 @@ case class ArrayContains(left: Expression, right: Expression)
14641464
nullSafeCodeGen(ctx, ev, (arr, value) => {
14651465
val i = ctx.freshName("i")
14661466
val getValue = CodeGenerator.getValue(arr, right.dataType, i)
1467-
s"""
1468-
for (int $i = 0; $i < $arr.numElements(); $i ++) {
1469-
if ($arr.isNullAt($i)) {
1470-
${ev.isNull} = true;
1471-
} else if (${ctx.genEqual(right.dataType, value, getValue)}) {
1472-
${ev.isNull} = false;
1473-
${ev.value} = true;
1474-
break;
1475-
}
1467+
val loopBodyCode = if (nullable) {
1468+
s"""
1469+
|if ($arr.isNullAt($i)) {
1470+
| ${ev.isNull} = true;
1471+
|} else if (${ctx.genEqual(right.dataType, value, getValue)}) {
1472+
| ${ev.isNull} = false;
1473+
| ${ev.value} = true;
1474+
| break;
1475+
|}
1476+
""".stripMargin
1477+
} else {
1478+
s"""
1479+
|if (${ctx.genEqual(right.dataType, value, getValue)}) {
1480+
| ${ev.value} = true;
1481+
| break;
1482+
|}
1483+
""".stripMargin
14761484
}
1477-
"""
1485+
s"""
1486+
|for (int $i = 0; $i < $arr.numElements(); $i ++) {
1487+
| $loopBodyCode
1488+
|}
1489+
""".stripMargin
14781490
})
14791491
}
14801492

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -383,10 +383,13 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
383383
val a3 = Literal.create(null, ArrayType(StringType))
384384
val a4 = Literal.create(Seq(create_row(1)), ArrayType(StructType(Seq(
385385
StructField("a", IntegerType, true)))))
386+
// Explicitly mark the array type not nullable (spark-25308)
387+
val a5 = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType, containsNull = false))
386388

387389
checkEvaluation(ArrayContains(a0, Literal(1)), true)
388390
checkEvaluation(ArrayContains(a0, Literal(0)), false)
389391
checkEvaluation(ArrayContains(a0, Literal.create(null, IntegerType)), null)
392+
checkEvaluation(ArrayContains(a5, Literal(1)), true)
390393

391394
checkEvaluation(ArrayContains(a1, Literal("")), true)
392395
checkEvaluation(ArrayContains(a1, Literal("a")), null)

0 commit comments

Comments
 (0)