Skip to content

Commit 70dbb75

Browse files
committed
chore: Fallback to Spark for array_reverse for binary
1 parent 4cfceb7 commit 70dbb75

File tree

2 files changed

+40
-1
lines changed

2 files changed

+40
-1
lines changed

spark/src/main/scala/org/apache/comet/serde/arrays.scala

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,25 @@ object CometGetArrayItem extends CometExpressionSerde[GetArrayItem] {
435435
}
436436

437437
object CometArrayReverse extends CometExpressionSerde[Reverse] with ArraysBase {
438+
val unsupportedReason = "reverse on array containing binary is not supported"
439+
440+
@tailrec
441+
private def containsBinary(dt: DataType): Boolean = {
442+
dt match {
443+
case BinaryType => true
444+
case ArrayType(elementType, _) => containsBinary(elementType)
445+
case _ => false
446+
}
447+
}
448+
449+
override def getSupportLevel(expr: Reverse): SupportLevel = {
450+
if (containsBinary(expr.child.dataType)) {
451+
Incompatible(Some(unsupportedReason))
452+
} else {
453+
Compatible(None)
454+
}
455+
}
456+
438457
override def convert(
439458
expr: Reverse,
440459
inputs: Seq[Attribute],
@@ -474,7 +493,7 @@ object CometElementAt extends CometExpressionSerde[ElementAt] {
474493
.setOneBased(true)
475494
.setFailOnError(expr.failOnError)
476495

477-
defaultExpr.foreach(arrayExtractBuilder.setDefaultValue(_))
496+
defaultExpr.foreach(arrayExtractBuilder.setDefaultValue)
478497

479498
Some(
480499
ExprOuterClass.Expr

spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -773,4 +773,24 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp
773773
}
774774
}
775775
}
776+
777+
test("array_reverse - fallback for binary array") {
778+
withTable("t1") {
779+
sql(
780+
"create table t1 using parquet as " +
781+
"select cast(null as array<binary>) c1, cast(array() as array<binary>) c2 from range(10)")
782+
783+
checkSparkAnswerAndFallbackReason(
784+
"select reverse(array(c1, c2)) AS x FROM t1",
785+
CometArrayReverse.unsupportedReason)
786+
787+
checkSparkAnswerAndFallbackReason(
788+
"select reverse(array(c1, c1)) AS x FROM t1",
789+
CometArrayReverse.unsupportedReason)
790+
791+
checkSparkAnswerAndFallbackReason(
792+
"select reverse(array(array(c1), array(c2))) AS x FROM t1",
793+
CometArrayReverse.unsupportedReason)
794+
}
795+
}
776796
}

0 commit comments

Comments
 (0)