ES|QL: Fix wrong pruning of plans with no output columns (#133405)

Author: luigidellaquila

docs/changelog/133405.yaml

@@ -0,0 +1,5 @@
+pr: 133405
+summary: Fix wrong pruning of plans with no output columns
+area: ES|QL
+type: bug
+issues: []

server/src/main/resources/transport/definitions/referable/esql_plan_with_no_columns.csv

@@ -0,0 +1 @@
+9186000

server/src/main/resources/transport/upper_bounds/9.3.csv

@@ -1 +1 @@
-initial_9.2.0,9185000
+esql_plan_with_no_columns,9186000

x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicPageTests.java

@@ -51,6 +51,15 @@ public void testEqualityAndHashCodeSmallInput() {
         );
         in.releaseBlocks();
 
+        in = new Page(10);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(
+            in,
+            page -> new Page(10),
+            page -> new Page(8, blockFactory.newConstantIntBlockWith(1, 8)),
+            Page::releaseBlocks
+        );
+        in.releaseBlocks();
+
         in = new Page(blockFactory.newIntArrayVector(new int[] {}, 0).asBlock());
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(
             in,
@@ -133,8 +142,8 @@ public void testEqualityAndHashCode() throws IOException {
             return new Page(blocks);
         };
 
-        int positions = randomIntBetween(1, 512);
-        int blockCount = randomIntBetween(1, 256);
+        int positions = randomIntBetween(0, 512);
+        int blockCount = randomIntBetween(0, 256);
         Block[] blocks = new Block[blockCount];
         for (int blockIndex = 0; blockIndex < blockCount; blockIndex++) {
             blocks[blockIndex] = switch (randomInt(9)) {
@@ -198,6 +207,16 @@ public void testAppend() {
         page2.releaseBlocks();
     }
 
+    public void testAppendToEmpty() {
+        Page page1 = new Page(10);
+        Page page2 = page1.appendBlock(blockFactory.newLongArrayVector(LongStream.range(0, 10).toArray(), 10).asBlock());
+        assertThat(0, is(page1.getBlockCount()));
+        assertThat(1, is(page2.getBlockCount()));
+        LongBlock block1 = page2.getBlock(0);
+        IntStream.range(0, 10).forEach(i -> assertThat((long) i, is(block1.getLong(i))));
+        page2.releaseBlocks();
+    }
+
     public void testPageSerializationSimple() throws IOException {
         IntVector toFilter = blockFactory.newIntArrayVector(IntStream.range(0, 20).toArray(), 20);
         Page origPage = new Page(
@@ -248,6 +267,22 @@ public void testPageSerializationSimple() throws IOException {
         }
     }
 
+    public void testPageSerializationEmpty() throws IOException {
+        Page origPage = new Page(10);
+        try {
+            Page deserPage = serializeDeserializePage(origPage);
+            try {
+                EqualsHashCodeTestUtils.checkEqualsAndHashCode(origPage, unused -> deserPage);
+                assertEquals(origPage.getBlockCount(), deserPage.getBlockCount());
+                assertEquals(origPage.getPositionCount(), deserPage.getPositionCount());
+            } finally {
+                deserPage.releaseBlocks();
+            }
+        } finally {
+            origPage.releaseBlocks();
+        }
+    }
+
     public void testSerializationListPages() throws IOException {
         final int positions = randomIntBetween(1, 64);
         List<Page> origPages = List.of(
@@ -265,7 +300,8 @@ public void testSerializationListPages() throws IOException {
                     positions
                 )
             ),
-            new Page(blockFactory.newConstantBytesRefBlockWith(new BytesRef("Hello World"), positions))
+            new Page(blockFactory.newConstantBytesRefBlockWith(new BytesRef("Hello World"), positions)),
+            new Page(10)
         );
         try {
             EqualsHashCodeTestUtils.checkEqualsAndHashCode(origPages, page -> {

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java

@@ -36,6 +36,7 @@
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.lucene.DataPartitioning;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.SuppressForbidden;
@@ -498,7 +499,11 @@ public static LogicalPlan emptySource() {
     }
 
     public static LogicalPlan localSource(BlockFactory blockFactory, List<Attribute> fields, List<Object> row) {
-        return new LocalRelation(Source.EMPTY, fields, LocalSupplier.of(BlockUtils.fromListRow(blockFactory, row)));
+        return new LocalRelation(
+            Source.EMPTY,
+            fields,
+            LocalSupplier.of(row.isEmpty() ? new Page(0) : new Page(BlockUtils.fromListRow(blockFactory, row)))
+        );
     }
 
     public static <T> T as(Object node, Class<T> type) {

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.xpack.esql.generator.command.CommandGenerator;
 import org.elasticsearch.xpack.esql.generator.command.pipe.ChangePointGenerator;
 import org.elasticsearch.xpack.esql.generator.command.pipe.DissectGenerator;
+import org.elasticsearch.xpack.esql.generator.command.pipe.DropAllGenerator;
 import org.elasticsearch.xpack.esql.generator.command.pipe.DropGenerator;
 import org.elasticsearch.xpack.esql.generator.command.pipe.EnrichGenerator;
 import org.elasticsearch.xpack.esql.generator.command.pipe.EvalGenerator;
@@ -63,6 +64,7 @@ public class EsqlQueryGenerator {
         ChangePointGenerator.INSTANCE,
         DissectGenerator.INSTANCE,
         DropGenerator.INSTANCE,
+        DropAllGenerator.INSTANCE,
         EnrichGenerator.INSTANCE,
         EvalGenerator.INSTANCE,
         ForkGenerator.INSTANCE,

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DropAllGenerator.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.generator.command.pipe;
+
+import org.elasticsearch.xpack.esql.generator.Column;
+import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator;
+import org.elasticsearch.xpack.esql.generator.QueryExecutor;
+import org.elasticsearch.xpack.esql.generator.command.CommandGenerator;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class DropAllGenerator implements CommandGenerator {
+
+    public static final String DROP_ALL = "drop_all";
+    public static final String DROPPED_COLUMNS = "dropped_columns";
+
+    public static final CommandGenerator INSTANCE = new DropAllGenerator();
+
+    @Override
+    public CommandDescription generate(
+        List<CommandDescription> previousCommands,
+        List<Column> previousOutput,
+        QuerySchema schema,
+        QueryExecutor executor
+    ) {
+        Set<String> droppedColumns = new HashSet<>();
+        String name = EsqlQueryGenerator.randomStringField(previousOutput);
+        if (name == null || name.isEmpty()) {
+            return CommandGenerator.EMPTY_DESCRIPTION;
+        }
+
+        String cmdString = " | keep " + name + " | drop " + name;
+        return new CommandDescription(DROP_ALL, this, cmdString, Map.ofEntries(Map.entry(DROPPED_COLUMNS, droppedColumns)));
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public ValidationResult validateOutput(
+        List<CommandDescription> previousCommands,
+        CommandDescription commandDescription,
+        List<Column> previousColumns,
+        List<List<Object>> previousOutput,
+        List<Column> columns,
+        List<List<Object>> output
+    ) {
+        if (commandDescription == EMPTY_DESCRIPTION) {
+            return VALIDATION_OK;
+        }
+
+        if (columns.size() > 0) {
+            return new ValidationResult(
+                false,
+                "Expecting no columns, got [" + columns.stream().map(Column::name).collect(Collectors.joining(", ")) + "]"
+            );
+        }
+
+        return VALIDATION_OK;
+    }
+
+}

x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec

@@ -49,29 +49,86 @@ b:integer | x:integer
 ;
 
 dropAllColumns
-from employees | keep height | drop height | eval x = 1;
+required_capability: fix_no_columns
+from employees | limit 4 | keep height | drop height | eval x = 1;
 
 x:integer
+1
+1
+1
+1
+;
+
+dropEvalKeep
+required_capability: fix_no_columns
+from employees | drop salary | eval salary = 1 | keep salary | limit 4;
+
+salary:integer
+1
+1
+1
+1
 ;
 
+
+dropEvalStats
+required_capability: fix_no_columns
+from mv_sample* | drop `client_ip`, message | eval  `event_duration` = "foo", @timestamp = 1 | stats max(@timestamp) by event_duration;
+
+max(@timestamp):integer | event_duration:keyword
+1                       | foo
+; 
+
+
+
+dropAllColumnsIndexPattern
+required_capability: fix_no_columns
+from emp* | drop languages | eval languages = 123 | keep languages | limit 4;
+
+languages:integer
+123
+123
+123
+123
+;
+
+
+dropAllColumnsWithMetadata
+required_capability: fix_no_columns
+from employees metadata _index | drop languages | eval languages = 123 | keep languages | limit 4;
+
+languages:integer
+123
+123
+123
+123
+;
+
+
 dropAllColumns_WithLimit
+required_capability: fix_no_columns
 from employees | keep height | drop height | eval x = 1 | limit 3;
 
 x:integer
+1
+1
+1
 ;
 
 dropAllColumns_WithCount
+required_capability: fix_no_columns
 from employees | keep height | drop height | eval x = 1 | stats c=count(x);
 
 c:long
-0
+100
 ;
 
 dropAllColumns_WithStats
+required_capability: fix_no_columns
 from employees | keep height | drop height | eval x = 1 | stats c=count(x), mi=min(x), s=sum(x);
 
-c:l|mi:i|s:l
-0  |null|null
+c:l  | mi:i | s:l
+100  | 1    | 100
 ;
 
 

x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec

@@ -5287,6 +5287,21 @@ null           | null             | corge          | null           | null
 null           | null             | fred           | null           | null
 ;
 
+lookupAfterDropAllColumns
+required_capability: fix_no_columns
+required_capability: join_lookup_v12
+FROM languages
+| DROP language_code
+| EVAL language_code = 3
+| LOOKUP JOIN languages_lookup ON language_code
+| LIMIT 3
+;
+
+ language_code:integer | language_name:keyword 
+3                      |Spanish
+3                      |Spanish
+3                      |Spanish
+;
 
 lookupJoinWithPushableFilterOnRight
 required_capability: join_lookup_v12

x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample.csv-spec

@@ -246,3 +246,13 @@ emp_no:integer
 10081
 // end::sampleForDocs-result[]
 ;
+
+
+sampleStatsEval
+required_capability: fix_no_columns
+required_capability: sample_v3
+FROM employees | SAMPLE 0.5 | LIMIT 10 | STATS count = COUNT() | EVAL is_expected = count > 0;
+
+count:long | is_expected:boolean
+10         | true
+;