Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
55 commits
Select commit Hold shift + click to select a range
69cfa82
Add `mv_contains_all` function
mjmbischoff Aug 18, 2025
e250247
[CI] Auto commit changes from spotless
Aug 19, 2025
f02c400
Enhance `mv_contains_all` function with additional examples and docum…
mjmbischoff Aug 19, 2025
4d0d5bc
[CI] Auto commit changes from spotless
Aug 19, 2025
dd44baa
Add documentation for `mv_contains_all` function
mjmbischoff Aug 19, 2025
fa3a339
Fixing documentation for `mv_contains_all` function
mjmbischoff Aug 19, 2025
05d64b6
Regenerated `mv_contains_all` documentation
mjmbischoff Aug 19, 2025
1a00c73
Update docs/changelog/133099.yaml
mjmbischoff Aug 19, 2025
d6f5e71
Update docs/changelog/133099.yaml
mjmbischoff Aug 19, 2025
8134f60
Merge branch 'main' into esql-MV_CONTAINS_ALL
mjmbischoff Aug 19, 2025
b4c31b9
Update docs/changelog/133099.yaml
mjmbischoff Aug 19, 2025
1a1b88d
Readding skipped markers as else the test gets run in the backwards c…
mjmbischoff Aug 19, 2025
88bd226
Merge branch 'main' into esql-MV_CONTAINS_ALL
mjmbischoff Aug 19, 2025
bf837d9
Merge branch 'main' into esql-MV_CONTAINS_ALL
mjmbischoff Aug 19, 2025
23544d4
Update `mv_contains_all` docs to reflect the correct version and add …
mjmbischoff Aug 20, 2025
a1e1ba3
[CI] Auto commit changes from spotless
Aug 20, 2025
fdfb149
Merge branch 'main' into esql-MV_CONTAINS_ALL
mjmbischoff Aug 21, 2025
cd7625e
Processing review, changing null handling
mjmbischoff Aug 23, 2025
979aac9
[CI] Auto commit changes from spotless
Aug 23, 2025
254fc59
Slightly more efficient null handling and fix null test logic
mjmbischoff Aug 23, 2025
b0e942a
[CI] Auto commit changes from spotless
Aug 23, 2025
fb7d7fd
Merge branch 'main' into esql-MV_CONTAINS_ALL
mjmbischoff Aug 23, 2025
b0c959d
- Renamed MV_CONTAINS_ALL to MV_CONTAINS
mjmbischoff Aug 24, 2025
fbe8a2b
[CI] Auto commit changes from spotless
Aug 24, 2025
f71848f
Update documentation to reflect `MV_CONTAINS_ALL` renaming to `MV_CON…
mjmbischoff Aug 24, 2025
5584a93
Fix `MvContains` javadoc
mjmbischoff Aug 25, 2025
bfa03d8
Add preview marker to documentation link
mjmbischoff Aug 25, 2025
b9a222b
I think checkstyle needs to be updated :D
mjmbischoff Aug 25, 2025
51733d1
Merge branch 'main' into esql-MV_CONTAINS_ALL
mjmbischoff Aug 25, 2025
8228c24
Merge branch 'main' into esql-MV_CONTAINS_ALL
mjmbischoff Aug 25, 2025
445e7bb
Replacing table and markdown comment block
mjmbischoff Aug 25, 2025
b7aba70
Merge branch 'main' into esql-MV_CONTAINS_ALL
mjmbischoff Aug 25, 2025
8a75b82
Merge branch 'main' into esql-MV_CONTAINS_ALL
mjmbischoff Aug 26, 2025
08cc9fa
Merge branch 'elastic:main' into esql-MV_CONTAINS_ALL
mjmbischoff Aug 27, 2025
277249a
Add memory usage reporting to `MvContains` evaluators
mjmbischoff Aug 27, 2025
e4b62de
Merge branch 'main' into esql-MV_CONTAINS_ALL
mjmbischoff Aug 27, 2025
a856360
Update docs/changelog/133636.yaml
mjmbischoff Aug 27, 2025
f4d3020
removing yaml from other PR
mjmbischoff Aug 28, 2025
1f7f430
WIP still has test failures.
mjmbischoff Aug 28, 2025
2393634
[CI] Auto commit changes from spotless
Aug 28, 2025
b3da482
It seems like the isNull operates on the row position, not the value …
mjmbischoff Aug 28, 2025
dcc1310
[CI] Auto commit changes from spotless
Aug 28, 2025
2aa3eec
Merge branch 'main' into esql-MV_CONTAINS_ALL
mjmbischoff Aug 28, 2025
aee4e35
Refactor `MvContains` null handling by reusing existing `IsNullEvalua…
mjmbischoff Aug 28, 2025
87ec7ab
Refactor `MvContains` null handling by reusing existing `IsNullEvalua…
mjmbischoff Aug 28, 2025
23b2aad
[CI] Auto commit changes from spotless
Aug 28, 2025
4aaac35
Refactor constant evaluators in `EvalOperator` by introducing dedicat…
mjmbischoff Aug 28, 2025
d676cef
[CI] Auto commit changes from spotless
Aug 28, 2025
ebcbfc7
Merge branch 'main' into esql-MV_CONTAINS_ALL
mjmbischoff Aug 28, 2025
66ba735
Extending test case to cover multirow null
mjmbischoff Aug 29, 2025
7492fad
Remove unused `process` method from `MvContains` class.
mjmbischoff Aug 29, 2025
23d92f0
Documenting changes done to copied test method due to inflexibility /…
mjmbischoff Aug 29, 2025
d33e333
Always true.
mjmbischoff Aug 29, 2025
3d7619b
Merge branch 'main' into esql-MV_CONTAINS_ALL
mjmbischoff Aug 29, 2025
a31922b
[CI] Auto commit changes from spotless
Aug 29, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -157,34 +157,114 @@ default boolean eagerEvalSafeInLazy() {
long baseRamBytesUsed();
}

public static final ExpressionEvaluator.Factory CONSTANT_NULL_FACTORY = new ExpressionEvaluator.Factory() {
private record ConstantNullEvaluator(DriverContext context) implements ExpressionEvaluator {
private static final String NAME = "ConstantNull";
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantNullEvaluator.class);

@Override
public Block eval(Page page) {
return context.blockFactory().newConstantNullBlock(page.getPositionCount());
}

@Override
public ExpressionEvaluator get(DriverContext driverContext) {
return new ExpressionEvaluator() {
@Override
public Block eval(Page page) {
return driverContext.blockFactory().newConstantNullBlock(page.getPositionCount());
}

@Override
public void close() {}

@Override
public String toString() {
return CONSTANT_NULL_NAME;
}

@Override
public long baseRamBytesUsed() {
return 0;
}
public void close() {}

@Override
public String toString() {
return NAME;
}

@Override
public long baseRamBytesUsed() {
return BASE_RAM_BYTES_USED;
}

record Factory() implements ExpressionEvaluator.Factory {
@Override
public ConstantNullEvaluator get(DriverContext context) {
return new ConstantNullEvaluator(context);
};

@Override
public String toString() {
return NAME;
}
};
}

public static final ExpressionEvaluator.Factory CONSTANT_NULL_FACTORY = new ConstantNullEvaluator.Factory();

private record ConstantTrueEvaluator(DriverContext context) implements ExpressionEvaluator {
private static final String NAME = "ConstantTrue";
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantTrueEvaluator.class);

@Override
public Block eval(Page page) {
return context.blockFactory().newConstantBooleanBlockWith(true, page.getPositionCount());
}

@Override
public void close() {}

@Override
public String toString() {
return CONSTANT_NULL_NAME;
return NAME;
}

@Override
public long baseRamBytesUsed() {
return BASE_RAM_BYTES_USED;
}
};
private static final String CONSTANT_NULL_NAME = "ConstantNull";

record Factory() implements ExpressionEvaluator.Factory {
@Override
public ConstantTrueEvaluator get(DriverContext context) {
return new ConstantTrueEvaluator(context);
};

@Override
public String toString() {
return NAME;
}
};
}

public static final ExpressionEvaluator.Factory CONSTANT_TRUE_FACTORY = new ConstantTrueEvaluator.Factory();

private record ConstantFalseEvaluator(DriverContext context) implements ExpressionEvaluator {
private static final String NAME = "ConstantFalse";
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantFalseEvaluator.class);

@Override
public Block eval(Page page) {
return context.blockFactory().newConstantBooleanBlockWith(false, page.getPositionCount());
}

@Override
public void close() {}

@Override
public String toString() {
return NAME;
}

@Override
public long baseRamBytesUsed() {
return BASE_RAM_BYTES_USED;
}

record Factory() implements ExpressionEvaluator.Factory {
@Override
public ConstantFalseEvaluator get(DriverContext context) {
return new ConstantFalseEvaluator(context);
};

@Override
public String toString() {
return NAME;
}
};
}

public static final ExpressionEvaluator.Factory CONSTANT_FALSE_FACTORY = new ConstantFalseEvaluator.Factory();
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle;
import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
import org.elasticsearch.xpack.esql.expression.function.Param;
import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNull;
import org.elasticsearch.xpack.esql.planner.PlannerUtils;

import java.io.IOException;
Expand Down Expand Up @@ -181,7 +182,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
var subsetType = PlannerUtils.toElementType(right().dataType());

if (subsetType == ElementType.NULL) {
return new ConstantBooleanTrueEvaluator();
return EvalOperator.CONSTANT_TRUE_FACTORY;
}

if (supersetType != ElementType.NULL && supersetType != subsetType) {
Expand All @@ -200,18 +201,13 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
case DOUBLE -> new MvContainsDoubleEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right()));
case INT -> new MvContainsIntEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right()));
case LONG -> new MvContainsLongEvaluator.Factory(source(), toEvaluator.apply(left()), toEvaluator.apply(right()));
case NULL -> new MvContainsNullSupersetEvaluator.Factory(source(), toEvaluator.apply(right()));
case NULL -> new IsNull.IsNullEvaluatorFactory(toEvaluator.apply(right()));
default -> throw EsqlIllegalArgumentException.illegalDataType(dataType());
};
}

static void process(BooleanBlock.Builder builder, int position, Block subset) {
final var valueCount = subset.getValueCount(position);
final var startIndex = subset.getFirstValueIndex(position);
for (int valueIndex = startIndex; valueIndex < startIndex + valueCount; valueIndex++) {
if (subset.isNull(valueIndex)) {
continue;
}
if (subset.isNull(position) == false) {
appendTo(builder, false);
return;
}
Expand Down Expand Up @@ -356,74 +352,6 @@ public String toString() {
}
}

/**
* Evaluator for when the lhs is null
* Like the ones below should be able to autogenerate as well when the generator is more flexible
*/
public static class MvContainsNullSupersetEvaluator implements EvalOperator.ExpressionEvaluator {
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(MvContainsNullSupersetEvaluator.class);
private final EvalOperator.ExpressionEvaluator subsetFieldEvaluator;
private final DriverContext driverContext;

public MvContainsNullSupersetEvaluator(EvalOperator.ExpressionEvaluator subsetFieldEvaluator, DriverContext driverContext) {
this.subsetFieldEvaluator = subsetFieldEvaluator;
this.driverContext = driverContext;
}

@Override
public Block eval(Page page) {
try (Block block = subsetFieldEvaluator.eval(page)) {
return eval(page.getPositionCount(), block);
}
}

public BooleanBlock eval(int positionCount, Block subsetBlock) {
try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
for (int p = 0; p < positionCount; p++) {
MvContains.process(result, p, subsetBlock);
}
return result.build();
}
}

@Override
public long baseRamBytesUsed() {
long baseRamBytesUsed = BASE_RAM_BYTES_USED;
baseRamBytesUsed += subsetFieldEvaluator.baseRamBytesUsed();
return baseRamBytesUsed;
}

@Override
public void close() {
Releasables.closeExpectNoException(subsetFieldEvaluator);
}

@Override
public String toString() {
return "MvContainsNullSupersetEvaluator[" + "subsetField=" + subsetFieldEvaluator + "]";
}

public static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
private final Source source;
private final EvalOperator.ExpressionEvaluator.Factory subsetFieldEvaluator;

public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory subsetField) {
this.source = source;
this.subsetFieldEvaluator = subsetField;
}

@Override
public MvContainsNullSupersetEvaluator get(DriverContext context) {
return new MvContainsNullSupersetEvaluator(subsetFieldEvaluator.get(context), context);
}

@Override
public String toString() {
return "MvContainsNullSupersetEvaluator[" + "subsetField=" + subsetFieldEvaluator + "]";
}
}
}

/**
* Currently {@code EvaluatorImplementer} generates:
* if (allBlocksAreNulls) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import org.elasticsearch.compute.data.Block;
import org.elasticsearch.compute.data.Page;
import org.elasticsearch.compute.operator.DriverContext;
import org.elasticsearch.compute.operator.EvalOperator;
import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
import org.elasticsearch.core.Releasables;
import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
import org.elasticsearch.xpack.esql.core.expression.Expression;
Expand Down Expand Up @@ -103,7 +103,7 @@ public Object fold(FoldContext ctx) {
}

@Override
public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
return new IsNullEvaluatorFactory(toEvaluator.apply(field()));
}

Expand Down Expand Up @@ -138,9 +138,9 @@ public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHand
return new NotQuery(source(), new ExistsQuery(source(), handler.nameOf(field())));
}

record IsNullEvaluatorFactory(EvalOperator.ExpressionEvaluator.Factory field) implements EvalOperator.ExpressionEvaluator.Factory {
public record IsNullEvaluatorFactory(ExpressionEvaluator.Factory field) implements ExpressionEvaluator.Factory {
@Override
public EvalOperator.ExpressionEvaluator get(DriverContext context) {
public ExpressionEvaluator get(DriverContext context) {
return new IsNullEvaluator(context, field.get(context));
}

Expand All @@ -150,9 +150,7 @@ public String toString() {
}
}

record IsNullEvaluator(DriverContext driverContext, EvalOperator.ExpressionEvaluator field)
implements
EvalOperator.ExpressionEvaluator {
record IsNullEvaluator(DriverContext driverContext, ExpressionEvaluator field) implements ExpressionEvaluator {
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(IsNullEvaluator.class);

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -341,9 +341,7 @@ protected static List<TestCaseSupplier> anyNullIsNull(
);
return new TestCaseSupplier.TestCase(
typeDataWithNull,
nullPosition == 0
? "MvContainsNullSupersetEvaluator[subsetField=Attribute[channel=1]]"
: "ConstantBooleanTrueEvaluator",
nullPosition == 0 ? "IsNullEvaluator[field=Attribute[channel=1]]" : "ConstantTrue",
expectedType.expectedType(nullPosition, DataType.BOOLEAN, originalTestCase),
equalTo(nullPosition == 1)
);
Expand Down
Loading