Merge branch 'main' into esql-MV_CONTAINS_ALL

Author: mjmbischoff

benchmarks/build.gradle

@@ -47,6 +47,7 @@ dependencies {
   api(project(':x-pack:plugin:core'))
   api(project(':x-pack:plugin:esql'))
   api(project(':x-pack:plugin:esql:compute'))
+  api(project(':x-pack:plugin:mapper-exponential-histogram'))
   implementation project(path: ':libs:native')
   implementation project(path: ':libs:simdvec')
   implementation project(path: ':libs:exponential-histogram')

benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java

@@ -9,11 +9,15 @@
 
 package org.elasticsearch.benchmark.exponentialhistogram;
 
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.exponentialhistogram.BucketIterator;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogramGenerator;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogramMerger;
+import org.elasticsearch.xpack.exponentialhistogram.CompressedExponentialHistogram;
+import org.elasticsearch.xpack.exponentialhistogram.IndexWithCount;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -27,6 +31,8 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
 import java.util.concurrent.ThreadLocalRandom;
@@ -47,6 +53,9 @@ public class ExponentialHistogramMergeBench {
     @Param({ "0.01", "0.1", "0.25", "0.5", "1.0", "2.0" })
     double mergedHistoSizeFactor;
 
+    @Param({ "array-backed", "compressed" })
+    String histoImplementation;
+
     Random random;
     ExponentialHistogramMerger histoMerger;
 
@@ -81,16 +90,54 @@ public void setUp() {
                 bucketIndex += 1 + random.nextInt(bucketCount) % (Math.max(1, bucketCount / dataPointSize));
                 generator.add(Math.pow(1.001, bucketIndex));
             }
-            toMerge[i] = generator.getAndClear();
-            cnt = getBucketCount(toMerge[i]);
+            ExponentialHistogram histogram = generator.getAndClear();
+            cnt = getBucketCount(histogram);
             if (cnt < dataPointSize) {
-                throw new IllegalArgumentException("Expected bucket count to be " + dataPointSize + ", but was " + cnt);
+                throw new IllegalStateException("Expected bucket count to be " + dataPointSize + ", but was " + cnt);
+            }
+
+            if ("array-backed".equals(histoImplementation)) {
+                toMerge[i] = histogram;
+            } else if ("compressed".equals(histoImplementation)) {
+                toMerge[i] = asCompressedHistogram(histogram);
+            } else {
+                throw new IllegalArgumentException("Unknown implementation: " + histoImplementation);
             }
         }
 
         index = 0;
     }
 
+    private ExponentialHistogram asCompressedHistogram(ExponentialHistogram histogram) {
+        List<IndexWithCount> negativeBuckets = new ArrayList<>();
+        List<IndexWithCount> positiveBuckets = new ArrayList<>();
+
+        BucketIterator it = histogram.negativeBuckets().iterator();
+        while (it.hasNext()) {
+            negativeBuckets.add(new IndexWithCount(it.peekIndex(), it.peekCount()));
+            it.advance();
+        }
+        it = histogram.positiveBuckets().iterator();
+        while (it.hasNext()) {
+            positiveBuckets.add(new IndexWithCount(it.peekIndex(), it.peekCount()));
+            it.advance();
+        }
+
+        long totalCount = histogram.zeroBucket().count() + histogram.negativeBuckets().valueCount() + histogram.positiveBuckets()
+            .valueCount();
+        BytesStreamOutput histoBytes = new BytesStreamOutput();
+        try {
+            CompressedExponentialHistogram.writeHistogramBytes(histoBytes, histogram.scale(), negativeBuckets, positiveBuckets);
+            CompressedExponentialHistogram result = new CompressedExponentialHistogram();
+            BytesRef data = histoBytes.bytes().toBytesRef();
+            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, data);
+            return result;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+    }
+
     private static int getBucketCount(ExponentialHistogram histo) {
         int cnt = 0;
         for (BucketIterator it : List.of(histo.negativeBuckets().iterator(), histo.positiveBuckets().iterator())) {

benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java

@@ -258,7 +258,7 @@ private static IndexWriterConfig createIndexWriterConfig(boolean optimizedMergeE
         );
         config.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
         config.setMergePolicy(new LogByteSizeMergePolicy());
-        var docValuesFormat = new ES819TSDBDocValuesFormat(4096, optimizedMergeEnabled);
+        var docValuesFormat = new ES819TSDBDocValuesFormat(4096, 512, optimizedMergeEnabled);
         config.setCodec(new Elasticsearch900Lucene101Codec() {
 
             @Override

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=443c9c8ee2ac1ee0e11881a40f2376d79c66386264a44b24a9f8ca67e633375f
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.2-all.zip
+distributionSha256Sum=f759b8dd5204e2e3fa4ca3e73f452f087153cf81bac9561eeb854229cc2c5365
+distributionUrl=https\://services.gradle.org/distributions/gradle-9.0.0-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy

@@ -53,8 +53,10 @@ abstract class AbstractRestResourcesFuncTest extends AbstractGradleFuncTest {
         }
         """
 
-        subProject(":distribution:archives:integ-test-zip") << "configurations.create('extracted')\n"
-        subProject(":distribution:archives:integ-test-zip") << "configurations.create('default')\n"
+        subProject(":distribution:archives:integ-test-zip") << """
+apply plugin: 'base'
+configurations.create('extracted')
+"""
     }
 
     void setupRestResources(List<String> apis, List<String> tests = [], List<String> xpackTests = []) {

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy

@@ -80,11 +80,8 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT
     def "registered distribution provides archives and directory variant"() {
         given:
         file('someFile.txt') << "some content"
-
-        settingsFile << """
-            include ':consumer'
-            include ':producer-tar'
-        """
+        subProject("consumer")
+        subProject("producer-tar")
 
         buildFile << """
         import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
@@ -154,9 +151,7 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT
     def "builds extracted distribution via extractedAssemble"() {
         given:
         file('someFile.txt') << "some content"
-        settingsFile << """
-            include ':producer-tar'
-        """
+        subProject("producer-tar")
 
         buildFile << """
         import org.gradle.api.artifacts.type.ArtifactTypeDefinition;

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy

@@ -117,5 +117,4 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
         result.output.contains("nested folder /distribution/bwc/minor/build/bwc/checkout-8.x/" +
                         "distribution/archives/darwin-tar/build/install/elasticsearch-8.4.0-SNAPSHOT")
     }
-
 }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTarFuncTest.groovy

@@ -9,15 +9,15 @@
 
 package org.elasticsearch.gradle.internal
 
-import spock.lang.Ignore
+
+import spock.lang.Unroll
 
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
 import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
 import org.gradle.api.GradleException
-import spock.lang.Unroll
 
 import java.nio.file.Files
 import java.nio.file.Path
@@ -66,8 +66,12 @@ tasks.register("buildBZip2Tar", SymbolicLinkPreservingTar) { SymbolicLinkPreserv
   from fileTree("archiveRoot")
 
   into('config') {
-    dirMode 0750
-    fileMode 0660
+    dirPermissions {
+        unix(0750)
+    }
+    filePermissions {
+        unix(0660)
+    }
     from "real-folder2"
   }
 }
@@ -118,8 +122,10 @@ tasks.register("buildTar", SymbolicLinkPreservingTar) { SymbolicLinkPreservingTa
         preserverTimestamp << [true, false]
     }
 
-    private boolean assertTar(final File archive, final Function<? super FileInputStream, ? extends InputStream> wrapper, boolean preserveFileTimestamps)
-            throws IOException {
+    private boolean assertTar(final File archive,
+                              final Function<? super FileInputStream, ? extends InputStream> wrapper,
+                              boolean preserveFileTimestamps)
+        throws IOException {
         try (TarArchiveInputStream tar = new TarArchiveInputStream(wrapper.apply(new FileInputStream(archive)))) {
             TarArchiveEntry entry = tar.getNextTarEntry();
             boolean realFolderEntry = false;
@@ -132,7 +138,7 @@ tasks.register("buildTar", SymbolicLinkPreservingTar) { SymbolicLinkPreservingTa
                 if (entry.getName().equals("real-folder/")) {
                     assert entry.isDirectory()
                     realFolderEntry = true
-                }  else if (entry.getName().equals("real-folder/file")) {
+                } else if (entry.getName().equals("real-folder/file")) {
                     assert entry.isFile()
                     fileEntry = true
                 } else if (entry.getName().equals("real-folder/link-to-file")) {
@@ -145,7 +151,7 @@ tasks.register("buildTar", SymbolicLinkPreservingTar) { SymbolicLinkPreservingTa
                 } else if (entry.getName().equals("config/sub/")) {
                     assert entry.isDirectory()
                     assert entry.getMode() == 16872
-                }else if (entry.getName().equals("link-in-folder/")) {
+                } else if (entry.getName().equals("link-in-folder/")) {
                     assert entry.isDirectory()
                     linkInFolderEntry = true
                 } else if (entry.getName().equals("link-in-folder/link-to-file")) {

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy

@@ -41,11 +41,11 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
         javaResource("myserver", "transport/definitions/unreferenced/" + name + ".csv", id)
     }
 
-    def definedAndUsedTransportVersion(String name, String ids) {
-        return definedAndUsedTransportVersion(name, ids, "Test${name.capitalize()}")
+    def namedAndReferencedTransportVersion(String name, String ids) {
+        return namedAndReferencedTransportVersion(name, ids, "Test${name.capitalize()}")
     }
 
-    def definedAndUsedTransportVersion(String name, String ids, String classname) {
+    def namedAndReferencedTransportVersion(String name, String ids, String classname) {
         javaSource("myserver", "org.elasticsearch", classname, "", """
             static final TransportVersion usage = TransportVersion.fromName("${name}");
         """)
@@ -60,17 +60,17 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
         return gradleRunner(":${project}:validateTransportVersionReferences").buildAndFail()
     }
 
-    def validateDefinitionsFails() {
-        return gradleRunner(":myserver:validateTransportVersionDefinitions").buildAndFail()
+    def validateResourcesFails() {
+        return gradleRunner(":myserver:validateTransportVersionResources").buildAndFail()
     }
 
-    def assertReferencesFailure(BuildResult result, String project, String expectedOutput) {
+    def assertValidateReferencesFailure(BuildResult result, String project, String expectedOutput) {
         result.task(":${project}:validateTransportVersionReferences").outcome == TaskOutcome.FAILED
         assertOutputContains(result.output, expectedOutput)
     }
 
-    def assertDefinitionsFailure(BuildResult result, String expectedOutput) {
-        result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.FAILED
+    def assertValidateResourcesFailure(BuildResult result, String expectedOutput) {
+        result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.FAILED
         assertOutputContains(result.output, expectedOutput)
     }
 
@@ -81,9 +81,6 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
             include ':myserver'
             include ':myplugin'
         """
-        file("gradle.properties") << """
-            org.elasticsearch.transport.definitionsProject=:myserver
-        """
 
         file("myserver/build.gradle") << """
             apply plugin: 'java-library'