chore: reenable array_contains tests (#3912)

Author: comphead

docs/source/user-guide/latest/compatibility.md

@@ -60,8 +60,6 @@ the [Comet Supported Expressions Guide](expressions.md) for more information on
 
 ### Array Expressions
 
-- **ArrayContains**: Returns null instead of false for empty arrays with literal values.
-  [#3346](https://github.com/apache/datafusion-comet/issues/3346)
 - **ArrayRemove**: Returns null when the element to remove is null, instead of removing null elements from the array.
   [#3173](https://github.com/apache/datafusion-comet/issues/3173)
 - **ArraysOverlap**: Inconsistent behavior when arrays contain NULL values.

docs/source/user-guide/latest/expressions.md

@@ -234,7 +234,7 @@ Comet supports using the following aggregate functions within window contexts wi
 | -------------- | ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | ArrayAppend    | Yes               |                                                                                                                                                                                           |
 | ArrayCompact   | No                |                                                                                                                                                                                           |
-| ArrayContains  | No                | Returns null instead of false for empty arrays with literal values ([#3346](https://github.com/apache/datafusion-comet/issues/3346))                                                      |
+| ArrayContains  | Yes               |                                                                                                                                                                                           |
 | ArrayDistinct  | No                | Behaves differently than spark. Comet first sorts then removes duplicates while Spark preserves the original order.                                                                       |
 | ArrayExcept    | No                |                                                                                                                                                                                           |
 | ArrayFilter    | Yes               | Only supports case where function is `IsNotNull`                                                                                                                                          |

native/core/src/execution/jni_api.rs

@@ -40,6 +40,7 @@ use datafusion::{
     prelude::{SessionConfig, SessionContext},
 };
 use datafusion_comet_proto::spark_operator::Operator;
+use datafusion_spark::function::array::array_contains::SparkArrayContains;
 use datafusion_spark::function::bitwise::bit_count::SparkBitCount;
 use datafusion_spark::function::bitwise::bit_get::SparkBitGet;
 use datafusion_spark::function::bitwise::bitwise_not::SparkBitwiseNot;
@@ -416,6 +417,7 @@ fn register_datafusion_spark_function(session_ctx: &SessionContext) {
     session_ctx.register_udf(ScalarUDF::new_from_impl(SparkLuhnCheck::default()));
     session_ctx.register_udf(ScalarUDF::new_from_impl(SparkSpace::default()));
     session_ctx.register_udf(ScalarUDF::new_from_impl(SparkBitCount::default()));
+    session_ctx.register_udf(ScalarUDF::new_from_impl(SparkArrayContains::default()));
 }
 
 /// Prepares arrow arrays for output.

spark/src/main/scala/org/apache/comet/serde/arrays.scala

@@ -141,49 +141,14 @@ object CometArrayAppend extends CometExpressionSerde[ArrayAppend] {
 
 object CometArrayContains extends CometExpressionSerde[ArrayContains] {
 
-  override def getSupportLevel(expr: ArrayContains): SupportLevel =
-    Incompatible(
-      Some(
-        "Returns null instead of false for empty arrays with literal values" +
-          " (https://github.com/apache/datafusion-comet/issues/3346)"))
-
   override def convert(
       expr: ArrayContains,
       inputs: Seq[Attribute],
       binding: Boolean): Option[ExprOuterClass.Expr] = {
     val arrayExprProto = exprToProto(expr.children.head, inputs, binding)
     val keyExprProto = exprToProto(expr.children(1), inputs, binding)
 
-    val arrayContainsScalarExpr =
-      scalarFunctionExprToProto("array_has", arrayExprProto, keyExprProto)
-
-    // Handle NULL array input - return NULL if array is NULL (matching Spark's behavior)
-    val isNotNullExpr = createUnaryExpr(
-      expr,
-      expr.children.head,
-      inputs,
-      binding,
-      (builder, unaryExpr) => builder.setIsNotNull(unaryExpr))
-
-    val nullLiteralProto = exprToProto(Literal(null, BooleanType), Seq.empty)
-
-    if (arrayContainsScalarExpr.isDefined && isNotNullExpr.isDefined &&
-      nullLiteralProto.isDefined) {
-      val caseWhenExpr = ExprOuterClass.CaseWhen
-        .newBuilder()
-        .addWhen(isNotNullExpr.get)
-        .addThen(arrayContainsScalarExpr.get)
-        .setElseExpr(nullLiteralProto.get)
-        .build()
-      Some(
-        ExprOuterClass.Expr
-          .newBuilder()
-          .setCaseWhen(caseWhenExpr)
-          .build())
-    } else {
-      withInfo(expr, expr.children: _*)
-      None
-    }
+    scalarFunctionExprToProto("array_contains", arrayExprProto, keyExprProto)
   }
 }
 

spark/src/test/resources/sql-tests/expressions/array/array_contains.sql

@@ -23,21 +23,19 @@ CREATE TABLE test_array_contains(arr array<int>, val int) USING parquet
 statement
 INSERT INTO test_array_contains VALUES (array(1, 2, 3), 2), (array(1, 2, 3), 4), (array(1, NULL, 3), NULL), (array(), 1), (NULL, 1)
 
-query spark_answer_only
+query
 SELECT array_contains(arr, val) FROM test_array_contains
 
 -- column + literal
-query ignore(https://github.com/apache/datafusion-comet/issues/3346)
+query
 SELECT array_contains(arr, 2) FROM test_array_contains
 
 -- literal + column
-query spark_answer_only
+query
 SELECT array_contains(array(1, 2, 3), val) FROM test_array_contains
 
 -- literal + literal
--- Note: array_contains(array(), 1) still has a bug (issue #3346) so we use spark_answer_only
--- The NULL array case (cast(NULL as array<int>)) was fixed in issue #3345
-query spark_answer_only
+query
 SELECT array_contains(array(1, 2, 3), 2), array_contains(array(1, 2, 3), 4), array_contains(array(), 1), array_contains(cast(NULL as array<int>), 1)
 
 -- Additional NULL array tests (issue #3345 fix verification)