Processing review, changing null handling

Author: mjmbischoff

x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec

@@ -1931,33 +1931,37 @@ l1:integer | l2:integer
 null       | 0
 ;
 
-mvContainsAll#[skip:-9.1.99,reason:new mvContainsall function added in 9.2]
+mvContainsAll
+required_capability: fn_mv_contains_all
 // tag::mv_contains_all[]
-ROW a = "a", b = ["a", "b", "c"]
-| EVAL a_subset_of_b = mv_contains_all(b, a)
+ROW set = ["a", "b", "c"], element = "a"
+| EVAL set_contains_element = mv_contains_all(set, element)
 // end::mv_contains_all[]
 ;
 
 // tag::mv_contains_all-result[]
-a:keyword | b:keyword | a_subset_of_b:boolean 
-a         | [a, b, c]    | true         
+set:keyword | element:keyword | set_contains_element:boolean 
+[a, b, c]   | a               | true         
 // end::mv_contains_all-result[]
 ;
 
-mvContainsAll_bothsides#[skip:-9.1.99,reason:new mvContainsall function added in 9.2]
+mvContainsAll_bothsides
+required_capability: fn_mv_contains_all
 // tag::mv_contains_all_bothsides[]
-ROW a = ["a","c"], b = ["a", "b", "c"]
-| EVAL a_subset_of_b = mv_contains_all(b, a)
+ROW setA = ["a","c"], setB = ["a", "b", "c"]
+| EVAL a_subset_of_b = mv_contains_all(setB, setA)
+| EVAL b_subset_of_a = mv_contains_all(setA, setB)
 // end::mv_contains_all_bothsides[]
 ;
 
 // tag::mv_contains_all_bothsides-result[]
-a:keyword | b:keyword | a_subset_of_b:boolean 
-[a, c]    | [a, b, c] | true   
+setA:keyword | setB:keyword | a_subset_of_b:boolean | b_subset_of_a:boolean 
+[a, c]       | [a, b, c]    | true                  | false
 // end::mv_contains_all_bothsides-result[]
 ;
 
-mvContainsAllCombinations#[skip:-9.1.99,reason:new mvContainsall function added in 9.2]
+mvContainsAllCombinations
+required_capability: fn_mv_contains_all
 
 ROW a = "a", b = ["a", "b", "c"], n = null
 | EVAL aa = mv_contains_all(a, a), 
@@ -1969,11 +1973,12 @@ ROW a = "a", b = ["a", "b", "c"], n = null
        nn = mv_contains_all(n,n)
 ;
 
-a:keyword | b:keyword | n:null | aa:boolean | bb:boolean   | ab:boolean | ba:boolean | na:boolean | an:boolean | nn:null
-a         | [a, b, c] | null   | true       | true         | false      | true       | null       | null      | null
+a:keyword | b:keyword | n:null | aa:boolean | bb:boolean   | ab:boolean | ba:boolean | na:boolean | an:boolean | nn:boolean
+a         | [a, b, c] | null   | true       | true         | false      | true       | false      | true       | true
 ;
 
-mvContainsAll_where#[skip:-9.1.99,reason:new mvContainsall function added in 9.2]
+mvContainsAll_where
+required_capability: fn_mv_contains_all
 // tag::mv_contains_all_where[]
 FROM airports
 | WHERE mv_contains_all(type, ["major","military"]) AND scalerank == 9

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -245,6 +245,13 @@ public enum Cap {
          */
         FN_MONTH_NAME,
 
+
+        /**
+         * support for MV_CONTAINS_ALL function
+         */
+        FN_MV_CONTAINS_ALL,
+
+
         /**
          * Fixes for multiple functions not serializing their source, and emitting warnings with wrong line number and text.
          */

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsAll.java

@@ -15,13 +15,17 @@
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BytesRefBlock;
 import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.LongBlock;
-import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
+import org.elasticsearch.core.Releasables;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FoldContext;
+import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -50,17 +54,16 @@ public class MvContainsAll extends BinaryScalarFunction implements EvaluatorMapp
         "MvContainsAll",
         MvContainsAll::new
     );
-    private DataType dataType;
 
     @FunctionInfo(
         returnType = "boolean",
-        description = "\"Checks if all values yielded by the second multivalue expression are present in the values yielded by "
-            + "the first multivalue expression. Returns a boolean, or null if either expression is null.",
+        description = "Checks if all values yielded by the second multivalue expression are present in the values yielded by "
+            + "the first multivalue expression. Returns a boolean. Null values are treated as an empty set.",
         examples = {
             @Example(file = "string", tag = "mv_contains_all"),
             @Example(file = "string", tag = "mv_contains_all_bothsides"),
-            @Example(file = "string", tag = "mv_contains_all_where"), },
-        appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") }
+            @Example(file = "string", tag = "mv_contains_all_where"),},
+        appliesTo = {@FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0")}
     )
     public MvContainsAll(
         Source source,
@@ -81,7 +84,7 @@ public MvContainsAll(
                 "long",
                 "text",
                 "unsigned_long",
-                "version" },
+                "version"},
             description = "Multivalue expression."
         ) Expression superset,
         @Param(
@@ -101,7 +104,7 @@ public MvContainsAll(
                 "long",
                 "text",
                 "unsigned_long",
-                "version" },
+                "version"},
             description = "Multivalue expression."
         ) Expression subset
     ) {
@@ -127,24 +130,24 @@ protected TypeResolution resolveType() {
         if (resolution.unresolved()) {
             return resolution;
         }
-        dataType = left().dataType() == DataType.NULL ? DataType.NULL : DataType.BOOLEAN;
         if (left().dataType() == DataType.NULL) {
-            dataType = right().dataType() == DataType.NULL ? DataType.NULL : DataType.BOOLEAN;
             return isRepresentableExceptCounters(right(), sourceText(), SECOND);
         }
         return isType(right(), t -> t.noText() == left().dataType().noText(), sourceText(), SECOND, left().dataType().noText().typeName());
     }
 
     @Override
     public DataType dataType() {
-        if (dataType == null) {
-            resolveType();
-        }
-        return dataType;
+        return DataType.BOOLEAN;
+    }
+
+    @Override
+    public Nullability nullable() {
+        return Nullability.FALSE;
     }
 
     @Override
-    protected BinaryScalarFunction replaceChildren(Expression newLeft, Expression newRight) {
+    protected MvContainsAll replaceChildren(Expression newLeft, Expression newRight) {
         return new MvContainsAll(source(), newLeft, newRight);
     }
 
@@ -162,7 +165,7 @@ public Object fold(FoldContext ctx) {
     public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
         var supersetType = PlannerUtils.toElementType(left().dataType());
         var subsetType = PlannerUtils.toElementType(right().dataType());
-        if (supersetType != subsetType) {
+        if (supersetType != ElementType.NULL && subsetType != ElementType.NULL && supersetType != subsetType) {
             throw new EsqlIllegalArgumentException(
                 "Incompatible data types for MvContainsAll, superset type({}) value({}) and subset type({}) value({}) don't match.",
                 supersetType,
@@ -177,11 +180,12 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
             case DOUBLE -> new MvContainsAllDoubleEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right()));
             case INT -> new MvContainsAllIntEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right()));
             case LONG -> new MvContainsAllLongEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right()));
-            case NULL -> EvalOperator.CONSTANT_NULL_FACTORY;
+            case NULL -> new MvContainsAllNullEvaluator(toEvaluator.apply(right()));
             default -> throw EsqlIllegalArgumentException.illegalDataType(dataType());
         };
     }
 
+
     @Evaluator(extraName = "Int")
     static void process(BooleanBlock.Builder builder, int position, IntBlock field1, IntBlock field2) {
         appendTo(builder, containsAll(field1, field2, position, IntBlock::getInt));
@@ -228,10 +232,10 @@ static void appendTo(BooleanBlock.Builder builder, Boolean bool) {
      * A block is considered a subset if the superset contains values that test equal for all the values in the subset, independent of
      * order. Duplicates are ignored in the sense that for each duplicate in the subset, we will search/match against the first/any value
      * in the superset.
+     *
      * @param superset block to check against
-     * @param subset block containing values that should be present in the other block.
-     * @return {@code true} if the given blocks are a superset and subset to each other, {@code false} if not and {@code null} if the subset
-     * or superset contains only null values.
+     * @param subset   block containing values that should be present in the other block.
+     * @return {@code true} if the given blocks are a superset and subset to each other, {@code false} if not.
      */
     static <BlockType extends Block, Type> Boolean containsAll(
         BlockType superset,
@@ -242,8 +246,8 @@ static <BlockType extends Block, Type> Boolean containsAll(
         if (superset == subset) {
             return true;
         }
-        if (subset.areAllValuesNull() || superset.areAllValuesNull()) {
-            return null;
+        if (subset.areAllValuesNull()) {
+            return true;
         }
 
         final var subsetCount = subset.getValueCount(position);
@@ -283,4 +287,26 @@ static <BlockType extends Block, Type> boolean hasValue(
     interface ValueExtractor<BlockType extends Block, Type> {
         Type extractValue(BlockType block, int position);
     }
+
+    private record MvContainsAllNullEvaluator(ExpressionEvaluator.Factory toEvaluator) implements ExpressionEvaluator.Factory {
+        @Override
+        public ExpressionEvaluator get(DriverContext context) {
+            return new ExpressionEvaluator() {
+                final ExpressionEvaluator subsetField = toEvaluator.get(context);
+
+                @Override
+                public Block eval(Page page) {
+                    try (Block block = subsetField.eval(page)) {
+                        var position = page.getPositionCount();
+                        return context.blockFactory().newConstantBooleanBlockWith(block.isNull(position), position);
+                    }
+                }
+
+                @Override
+                public void close() {
+                    Releasables.closeExpectNoException(subsetField);
+                }
+            };
+        }
+    }
 }