Merge remote-tracking branch 'origin/tsdb/bulk-load-float' into tsdb/bulk-load-float

Author: kkrik-es

.github/workflows/docs-build.yml

@@ -13,6 +13,7 @@ jobs:
     with:
       path-pattern: docs/**
       path-pattern-ignore: docs/changelog/**/*.yaml
+      enable-cumulative-comment: true
     permissions:
       deployments: write
       id-token: write

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java

@@ -666,6 +666,11 @@ public Block eval(Page page) {
                 return mask;
             }
 
+            @Override
+            public long baseRamBytesUsed() {
+                return 0;
+            }
+
             @Override
             public void close() {
                 mask.close();

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -144,7 +144,7 @@ static void selfTest() {
     public String operation;
 
     private static Operator operator(String operation) {
-        return new EvalOperator(driverContext.blockFactory(), evaluator(operation));
+        return new EvalOperator(driverContext, evaluator(operation));
     }
 
     private static EvalOperator.ExpressionEvaluator evaluator(String operation) {

distribution/build.gradle

@@ -629,22 +629,6 @@ subprojects {
   }
 }
 
-['archives:windows-zip',
- 'archives:darwin-tar',
- 'archives:darwin-aarch64-tar',
- 'archives:linux-aarch64-tar',
- 'archives:linux-tar',
- 'archives:integ-test-zip',
- 'packages:rpm', 'packages:deb',
- 'packages:aarch64-rpm', 'packages:aarch64-deb',
-].forEach { subName ->
-  Project subproject = project("${project.path}:${subName}")
-  Configuration configuration = configurations.create(subproject.name)
-  dependencies {
-    "${configuration.name}" project(path: subproject.path, configuration:'default')
-  }
-}
-
 // This artifact makes it possible for other projects to pull
 // in the final log4j2.properties configuration, as it appears in the
 // archive distribution.

docs/changelog/133392.yaml

@@ -0,0 +1,5 @@
+pr: 133392
+summary: Track memory in evaluators
+area: ES|QL
+type: bug
+issues: []

muted-tests.yml

@@ -675,6 +675,18 @@ tests:
 - class: org.elasticsearch.xpack.logsdb.qa.LogsDbVersusReindexedLogsDbChallengeRestIT
   method: testEsqlSource
   issue: https://github.com/elastic/elasticsearch/issues/133520
+- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
+  method: testTopNPushedToLucene
+  issue: https://github.com/elastic/elasticsearch/issues/133556
+- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
+  method: testFilterNestedFields
+  issue: https://github.com/elastic/elasticsearch/issues/133557
+- class: org.elasticsearch.xpack.logsdb.qa.LogsDbVersusReindexedLogsDbChallengeRestIT
+  method: testMatchAllQuery
+  issue: https://github.com/elastic/elasticsearch/issues/133560
+- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
+  method: test {p0=search/10_source_filtering/no filtering}
+  issue: https://github.com/elastic/elasticsearch/issues/133561
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesProducer.java

@@ -1747,6 +1747,7 @@ private static class TermsDictEntry {
 
     static final class SingletonLongToSingletonOrdinalDelegate implements BlockLoader.SingletonLongBuilder {
         private final BlockLoader.SingletonOrdinalsBuilder builder;
+        private final int[] buffer = new int[ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
 
         SingletonLongToSingletonOrdinalDelegate(BlockLoader.SingletonOrdinalsBuilder builder) {
             this.builder = builder;
@@ -1759,20 +1760,21 @@ public BlockLoader.SingletonLongBuilder appendLong(long value) {
 
         @Override
         public BlockLoader.SingletonLongBuilder appendLongs(long[] values, int from, int length) {
+            assert length <= buffer.length;
             // Unfortunately, no array copy here...
             // Since we need to loop here, let's also keep track of min/max.
             int minOrd = Integer.MAX_VALUE;
             int maxOrd = Integer.MIN_VALUE;
             int counter = 0;
-            int[] convertedOrds = new int[length];
             int end = from + length;
             for (int j = from; j < end; j++) {
                 int ord = Math.toIntExact(values[j]);
-                convertedOrds[counter++] = ord;
+                buffer[counter++] = ord;
                 minOrd = Math.min(minOrd, ord);
                 maxOrd = Math.max(maxOrd, ord);
             }
-            builder.appendOrds(convertedOrds, 0, length, minOrd, maxOrd);
+            assert counter == length;
+            builder.appendOrds(buffer, 0, length, minOrd, maxOrd);
             return this;
         }
 

test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java

@@ -1522,7 +1522,7 @@ public void testSingletonLongBulkBlockReading() throws IOException {
     }
 
     public void testSingletonDoubleBulkBlockReading() throws IOException {
-        assumeTrue("field type supports bulk singleton long reading", supportsBulkDoubleBlockReading());
+        assumeTrue("field type supports bulk singleton double reading", supportsBulkDoubleBlockReading());
         testSingletonBulkBlockReading(columnAtATimeReader -> (BlockDocValuesReader.SingletonDoubles) columnAtATimeReader);
     }
 

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java

@@ -6,7 +6,9 @@
  */
 package org.elasticsearch.xpack.esql.core.expression;
 
+import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -35,7 +37,9 @@
 /**
  * Literal or constant.
  */
-public class Literal extends LeafExpression {
+public class Literal extends LeafExpression implements Accountable {
+    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Literal.class);
+
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
         Expression.class,
         "Literal",
@@ -169,6 +173,17 @@ public String nodeString() {
         return toString() + "[" + dataType + "]";
     }
 
+    @Override
+    public long ramBytesUsed() {
+        long ramBytesUsed = BASE_RAM_BYTES_USED;
+        if (value instanceof BytesRef b) {
+            ramBytesUsed += b.length;
+        } else {
+            ramBytesUsed += RamUsageEstimator.sizeOfObject(value);
+        }
+        return ramBytesUsed;
+    }
+
     /**
      * Utility method for creating a literal out of a foldable expression.
      * Throws an exception if the expression is not foldable.

x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/ConvertEvaluatorImplementer.java

@@ -23,6 +23,7 @@
 import javax.lang.model.type.TypeMirror;
 import javax.lang.model.util.Elements;
 
+import static org.elasticsearch.compute.gen.EvaluatorImplementer.baseRamBytesUsed;
 import static org.elasticsearch.compute.gen.Methods.buildFromFactory;
 import static org.elasticsearch.compute.gen.Methods.getMethod;
 import static org.elasticsearch.compute.gen.Types.ABSTRACT_CONVERT_FUNCTION_EVALUATOR;
@@ -98,6 +99,7 @@ private TypeSpec type() {
         builder.addJavadoc("This class is generated. Edit {@code " + getClass().getSimpleName() + "} instead.");
         builder.addModifiers(Modifier.PUBLIC, Modifier.FINAL);
         builder.superclass(ABSTRACT_CONVERT_FUNCTION_EVALUATOR);
+        builder.addField(baseRamBytesUsed(implementation));
 
         for (EvaluatorImplementer.ProcessFunctionArg a : processFunction.args) {
             a.declareField(builder);
@@ -113,6 +115,7 @@ private TypeSpec type() {
         }
         builder.addMethod(processFunction.toStringMethod(implementation));
         builder.addMethod(processFunction.close());
+        builder.addMethod(processFunction.baseRamBytesUsed());
         builder.addType(factory());
         return builder.build();
     }