Merge remote-tracking branch 'origin/main' into separate_weights_per_tier_v2

Author: nicktindall

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OptimizedScalarQuantizerBenchmark.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.vector;
+
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.index.codec.vectors.OptimizedScalarQuantizer;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(value = 3)
+public class OptimizedScalarQuantizerBenchmark {
+    static {
+        LogConfigurator.configureESLogging(); // native access requires logging to be initialized
+    }
+    @Param({ "384", "702", "1024" })
+    int dims;
+
+    float[] vector;
+    float[] centroid;
+    byte[] destination;
+
+    @Param({ "1", "4", "7" })
+    byte bits;
+
+    OptimizedScalarQuantizer osq = new OptimizedScalarQuantizer(VectorSimilarityFunction.DOT_PRODUCT);
+
+    @Setup(Level.Iteration)
+    public void init() {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        // random byte arrays for binary methods
+        destination = new byte[dims];
+        vector = new float[dims];
+        centroid = new float[dims];
+        for (int i = 0; i < dims; ++i) {
+            vector[i] = random.nextFloat();
+            centroid[i] = random.nextFloat();
+        }
+    }
+
+    @Benchmark
+    public byte[] scalar() {
+        osq.scalarQuantize(vector, destination, bits, centroid);
+        return destination;
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public byte[] vector() {
+        osq.scalarQuantize(vector, destination, bits, centroid);
+        return destination;
+    }
+}

build-tools-internal/version.properties

@@ -17,7 +17,6 @@ jna               = 5.12.1
 netty             = 4.1.118.Final
 commons_lang3     = 3.9
 google_oauth_client = 1.34.1
-awsv1sdk            = 1.12.746
 awsv2sdk            = 2.30.38
 reactive_streams    = 1.0.4
 

distribution/docker/build.gradle

@@ -590,7 +590,7 @@ for (final Architecture architecture : Architecture.values()) {
     }
     if(base == DockerBase.DEFAULT) {
       // Add additional docker hub specific context which we use solely for publishing to docker hub.
-      // At the moment it only differs in not labels added that we need for openshift certification
+      // At the moment it is exactly the same as the default context.
       addBuildDockerContextTask(architecture, base, 'DockerHubContext', "docker-hub-build-context")
     }
   }

distribution/docker/src/docker/Dockerfile.default

@@ -139,15 +139,13 @@ LABEL org.label-schema.build-date="${build_date}" \\
   org.opencontainers.image.vendor="Elastic" \\
   org.opencontainers.image.version="${version}"
 
-<% if (docker_context != 'docker-hub-build-context') { %>
 LABEL name="Elasticsearch" \\
   maintainer="[email protected]" \\
   vendor="Elastic" \\
   version="${version}" \\
   release="1" \\
   summary="Elasticsearch" \\
   description="You know, for search."
-<% } %>
 
 RUN mkdir /licenses && ln LICENSE.txt /licenses/LICENSE
 

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -61,6 +61,13 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 "-Dio.netty.noUnsafe=true",
                 "-Dio.netty.noKeySetOptimization=true",
                 "-Dio.netty.recycler.maxCapacityPerThread=0",
+                // temporary until we get off-heap vector stats in Lucene 10.3
+                "--add-opens=org.apache.lucene.core/org.apache.lucene.codecs.lucene99=org.elasticsearch.server",
+                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene90=org.elasticsearch.server",
+                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene91=org.elasticsearch.server",
+                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene92=org.elasticsearch.server",
+                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene94=org.elasticsearch.server",
+                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene95=org.elasticsearch.server",
                 // log4j 2
                 "-Dlog4j.shutdownHookEnabled=false",
                 "-Dlog4j2.disable.jmx=true",

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java

@@ -38,7 +38,6 @@
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 
-@ESTestCase.WithoutSecurityManager
 public class APMJvmOptionsTests extends ESTestCase {
 
     private Path installDir;

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmErgonomicsTests.java

@@ -15,7 +15,6 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.node.NodeRoleSettings;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.ESTestCase.WithoutSecurityManager;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -41,7 +40,6 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-@WithoutSecurityManager
 @SuppressFileSystems("*")
 public class JvmErgonomicsTests extends ESTestCase {
 

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java

@@ -15,7 +15,6 @@
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.ESTestCase.WithoutSecurityManager;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -43,7 +42,6 @@
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.not;
 
-@WithoutSecurityManager
 @LuceneTestCase.SuppressFileSystems("*")
 public class JvmOptionsParserTests extends ESTestCase {
 

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/MachineDependentHeapTests.java

@@ -11,7 +11,6 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.ESTestCase.WithoutSecurityManager;
 import org.hamcrest.Matcher;
 
 import java.util.Collections;
@@ -21,7 +20,6 @@
 import static org.hamcrest.Matchers.empty;
 
 // TODO: rework these tests to mock jvm option finder so they can run with security manager, no forking needed
-@WithoutSecurityManager
 public class MachineDependentHeapTests extends ESTestCase {
 
     public void testDefaultHeapSize() throws Exception {

docs/changelog/125570.yaml

@@ -0,0 +1,5 @@
+pr: 125570
+summary: ES|QL random sampling
+area: Machine Learning
+type: feature
+issues: []