merge

Author: mhl-b

benchmarks/build.gradle

@@ -47,6 +47,7 @@ dependencies {
   api(project(':x-pack:plugin:core'))
   api(project(':x-pack:plugin:esql'))
   api(project(':x-pack:plugin:esql:compute'))
+  api(project(':x-pack:plugin:mapper-exponential-histogram'))
   implementation project(path: ':libs:native')
   implementation project(path: ':libs:simdvec')
   implementation project(path: ':libs:exponential-histogram')

benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java

@@ -9,11 +9,15 @@
 
 package org.elasticsearch.benchmark.exponentialhistogram;
 
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.exponentialhistogram.BucketIterator;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogramGenerator;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogramMerger;
+import org.elasticsearch.xpack.exponentialhistogram.CompressedExponentialHistogram;
+import org.elasticsearch.xpack.exponentialhistogram.IndexWithCount;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -27,6 +31,8 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
 import java.util.concurrent.ThreadLocalRandom;
@@ -47,6 +53,9 @@ public class ExponentialHistogramMergeBench {
     @Param({ "0.01", "0.1", "0.25", "0.5", "1.0", "2.0" })
     double mergedHistoSizeFactor;
 
+    @Param({ "array-backed", "compressed" })
+    String histoImplementation;
+
     Random random;
     ExponentialHistogramMerger histoMerger;
 
@@ -81,16 +90,54 @@ public void setUp() {
                 bucketIndex += 1 + random.nextInt(bucketCount) % (Math.max(1, bucketCount / dataPointSize));
                 generator.add(Math.pow(1.001, bucketIndex));
             }
-            toMerge[i] = generator.getAndClear();
-            cnt = getBucketCount(toMerge[i]);
+            ExponentialHistogram histogram = generator.getAndClear();
+            cnt = getBucketCount(histogram);
             if (cnt < dataPointSize) {
-                throw new IllegalArgumentException("Expected bucket count to be " + dataPointSize + ", but was " + cnt);
+                throw new IllegalStateException("Expected bucket count to be " + dataPointSize + ", but was " + cnt);
+            }
+
+            if ("array-backed".equals(histoImplementation)) {
+                toMerge[i] = histogram;
+            } else if ("compressed".equals(histoImplementation)) {
+                toMerge[i] = asCompressedHistogram(histogram);
+            } else {
+                throw new IllegalArgumentException("Unknown implementation: " + histoImplementation);
             }
         }
 
         index = 0;
     }
 
+    private ExponentialHistogram asCompressedHistogram(ExponentialHistogram histogram) {
+        List<IndexWithCount> negativeBuckets = new ArrayList<>();
+        List<IndexWithCount> positiveBuckets = new ArrayList<>();
+
+        BucketIterator it = histogram.negativeBuckets().iterator();
+        while (it.hasNext()) {
+            negativeBuckets.add(new IndexWithCount(it.peekIndex(), it.peekCount()));
+            it.advance();
+        }
+        it = histogram.positiveBuckets().iterator();
+        while (it.hasNext()) {
+            positiveBuckets.add(new IndexWithCount(it.peekIndex(), it.peekCount()));
+            it.advance();
+        }
+
+        long totalCount = histogram.zeroBucket().count() + histogram.negativeBuckets().valueCount() + histogram.positiveBuckets()
+            .valueCount();
+        BytesStreamOutput histoBytes = new BytesStreamOutput();
+        try {
+            CompressedExponentialHistogram.writeHistogramBytes(histoBytes, histogram.scale(), negativeBuckets, positiveBuckets);
+            CompressedExponentialHistogram result = new CompressedExponentialHistogram();
+            BytesRef data = histoBytes.bytes().toBytesRef();
+            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, data);
+            return result;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+    }
+
     private static int getBucketCount(ExponentialHistogram histo) {
         int cnt = 0;
         for (BucketIterator it : List.of(histo.negativeBuckets().iterator(), histo.positiveBuckets().iterator())) {

benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java

@@ -258,7 +258,7 @@ private static IndexWriterConfig createIndexWriterConfig(boolean optimizedMergeE
         );
         config.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
         config.setMergePolicy(new LogByteSizeMergePolicy());
-        var docValuesFormat = new ES819TSDBDocValuesFormat(4096, optimizedMergeEnabled);
+        var docValuesFormat = new ES819TSDBDocValuesFormat(4096, 512, optimizedMergeEnabled);
         config.setCodec(new Elasticsearch900Lucene101Codec() {
 
             @Override

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java

@@ -34,12 +34,13 @@ public void apply(Project project) {
                 || buildParams.getBwcVersions().unreleasedInfo(version) == null
         );
 
-        if (shouldConfigureTestClustersWithOneProcessor()) {
-            NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
-                .getExtensions()
-                .getByName(TestClustersPlugin.EXTENSION_NAME);
-            testClusters.configureEach(elasticsearchCluster -> elasticsearchCluster.setting("node.processors", "1"));
-        }
+        NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
+            .getExtensions()
+            .getByName(TestClustersPlugin.EXTENSION_NAME);
+        // Limit the number of allocated processors for all nodes to 2 in the cluster by default.
+        // This is to ensure that the tests run consistently across different environments.
+        String processorCount = shouldConfigureTestClustersWithOneProcessor() ? "1" : "2";
+        testClusters.configureEach(elasticsearchCluster -> elasticsearchCluster.setting("node.processors", processorCount));
     }
 
     private boolean shouldConfigureTestClustersWithOneProcessor() {

build-tools/src/main/java/org/elasticsearch/gradle/LazyFileOutputStream.java

@@ -3,8 +3,7 @@
  * or more contributor license agreements. Licensed under the "Elastic License
  * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
  * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * your election, the "Server Side Public License v3.0 only", or the "Server Side Public License, v 1".
  */
 
 package org.elasticsearch.gradle;
@@ -18,48 +17,58 @@
  * An outputstream to a File that is lazily opened on the first write.
  */
 class LazyFileOutputStream extends OutputStream {
-    private OutputStream delegate;
+    private final File file;
+    private volatile OutputStream delegate;
+    private volatile boolean initialized = false;
+    private final Object lock = new Object();
 
     LazyFileOutputStream(File file) {
-        // use an initial dummy delegate to avoid doing a conditional on every write
-        this.delegate = new OutputStream() {
-            private void bootstrap() throws IOException {
-                file.getParentFile().mkdirs();
-                delegate = new FileOutputStream(file);
-            }
-
-            @Override
-            public void write(int b) throws IOException {
-                bootstrap();
-                delegate.write(b);
-            }
-
-            @Override
-            public void write(byte b[], int off, int len) throws IOException {
-                bootstrap();
-                delegate.write(b, off, len);
-            }
+        this.file = file;
+    }
 
-            @Override
-            public void write(byte b[]) throws IOException {
-                bootstrap();
-                delegate.write(b);
+    private void ensureInitialized() throws IOException {
+        if (initialized == false) {
+            synchronized (lock) {
+                if (initialized == false) {
+                    file.getParentFile().mkdirs();
+                    delegate = new FileOutputStream(file);
+                    initialized = true;
+                }
             }
-        };
+        }
     }
 
     @Override
     public void write(int b) throws IOException {
+        ensureInitialized();
         delegate.write(b);
     }
 
     @Override
     public void write(byte b[], int off, int len) throws IOException {
+        ensureInitialized();
         delegate.write(b, off, len);
     }
 
+    @Override
+    public void write(byte b[]) throws IOException {
+        ensureInitialized();
+        delegate.write(b);
+    }
+
     @Override
     public void close() throws IOException {
-        delegate.close();
+        synchronized (lock) {
+            if (initialized && delegate != null) {
+                delegate.close();
+            }
+        }
+    }
+
+    @Override
+    public void flush() throws IOException {
+        if (initialized && delegate != null) {
+            delegate.flush();
+        }
     }
 }

docs/changelog/133018.yaml

@@ -0,0 +1,5 @@
+pr: 133018
+summary: Add ordinal range encode for tsid
+area: TSDB
+type: enhancement
+issues: []

docs/changelog/133113.yaml

@@ -0,0 +1,5 @@
+pr: 133113
+summary: Limit the depth of a filter
+area: Infra/REST API
+type: enhancement
+issues: []

docs/changelog/133188.yaml

@@ -0,0 +1,6 @@
+pr: 133188
+summary: Don't fail search if bottom doc can't be formatted
+area: Search
+type: bug
+issues:
+ - 125321

docs/changelog/133193.yaml

@@ -0,0 +1,5 @@
+pr: 133193
+summary: Fix offset handling in Murmur3Hasher
+area: Infra/Core
+type: bug
+issues: []

docs/release-notes/breaking-changes.md

@@ -111,7 +111,7 @@ Existing `repository-s3` configurations may no longer be compatible. Notable dif
 - AWS SDK v2 requires the use of the V4 signature algorithm, therefore, the `s3.client.${CLIENT_NAME}.signer_override` setting is deprecated and no longer has any effect.
 - AWS SDK v2 does not support the `log-delivery-write` canned ACL.
 - AWS SDK v2 counts 4xx responses differently in its metrics reporting.
-- AWS SDK v2 always uses the regional STS endpoint, whereas AWS SDK v2 could use either a regional endpoint or the global `https://sts.amazonaws.com` one.
+- AWS SDK v2 always uses the regional STS endpoint, whereas AWS SDK v1 could use either a regional endpoint or the global `https://sts.amazonaws.com` one.
 
 **Action:**
 Test the upgrade in a non-production environment. Adapt your configuration to the new SDK functionality. This includes, but may not be limited to, the following items: