Merge branch 'main' into enh/inlinejoin_prune

Author: bpintea

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/Int4ScorerBenchmark.java

@@ -8,12 +8,14 @@
  */
 package org.elasticsearch.benchmark.vector;
 
+import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.util.VectorUtil;
+import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.simdvec.ES91Int4VectorsScorer;
@@ -52,20 +54,26 @@ public class Int4ScorerBenchmark {
         LogConfigurator.configureESLogging(); // native access requires logging to be initialized
     }
 
-    @Param({ "384", "702", "1024" })
+    @Param({ "384", "782", "1024" })
     int dims;
 
-    int numVectors = 200;
-    int numQueries = 10;
+    int numVectors = 20 * ES91Int4VectorsScorer.BULK_SIZE;
+    int numQueries = 5;
 
     byte[] scratch;
     byte[][] binaryVectors;
     byte[][] binaryQueries;
+    float[] scores = new float[ES91Int4VectorsScorer.BULK_SIZE];
+
+    float[] scratchFloats = new float[3];
 
     ES91Int4VectorsScorer scorer;
     Directory dir;
     IndexInput in;
 
+    OptimizedScalarQuantizer.QuantizationResult queryCorrections;
+    float centroidDp;
+
     @Setup
     public void setup() throws IOException {
         binaryVectors = new byte[numVectors][dims];
@@ -77,9 +85,19 @@ public void setup() throws IOException {
                     binaryVector[i] = (byte) ThreadLocalRandom.current().nextInt(16);
                 }
                 out.writeBytes(binaryVector, 0, binaryVector.length);
+                ThreadLocalRandom.current().nextBytes(binaryVector);
+                out.writeBytes(binaryVector, 0, 14); // corrections
             }
         }
 
+        queryCorrections = new OptimizedScalarQuantizer.QuantizationResult(
+            ThreadLocalRandom.current().nextFloat(),
+            ThreadLocalRandom.current().nextFloat(),
+            ThreadLocalRandom.current().nextFloat(),
+            Short.toUnsignedInt((short) ThreadLocalRandom.current().nextInt())
+        );
+        centroidDp = ThreadLocalRandom.current().nextFloat();
+
         in = dir.openInput("vectors", IOContext.DEFAULT);
         binaryQueries = new byte[numVectors][dims];
         for (byte[] binaryVector : binaryVectors) {
@@ -105,18 +123,66 @@ public void scoreFromArray(Blackhole bh) throws IOException {
             in.seek(0);
             for (int i = 0; i < numVectors; i++) {
                 in.readBytes(scratch, 0, dims);
-                bh.consume(VectorUtil.int4DotProduct(binaryQueries[j], scratch));
+                int dp = VectorUtil.int4DotProduct(binaryQueries[j], scratch);
+                in.readFloats(scratchFloats, 0, 3);
+                float score = scorer.applyCorrections(
+                    queryCorrections.lowerInterval(),
+                    queryCorrections.upperInterval(),
+                    queryCorrections.quantizedComponentSum(),
+                    queryCorrections.additionalCorrection(),
+                    VectorSimilarityFunction.EUCLIDEAN,
+                    centroidDp, // assuming no centroid dot product for this benchmark
+                    scratchFloats[0],
+                    scratchFloats[1],
+                    Short.toUnsignedInt(in.readShort()),
+                    scratchFloats[2],
+                    dp
+                );
+                bh.consume(score);
             }
         }
     }
 
     @Benchmark
     @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
-    public void scoreFromMemorySegmentOnlyVector(Blackhole bh) throws IOException {
+    public void scoreFromMemorySegment(Blackhole bh) throws IOException {
         for (int j = 0; j < numQueries; j++) {
             in.seek(0);
             for (int i = 0; i < numVectors; i++) {
-                bh.consume(scorer.int4DotProduct(binaryQueries[j]));
+                bh.consume(
+                    scorer.score(
+                        binaryQueries[j],
+                        queryCorrections.lowerInterval(),
+                        queryCorrections.upperInterval(),
+                        queryCorrections.quantizedComponentSum(),
+                        queryCorrections.additionalCorrection(),
+                        VectorSimilarityFunction.EUCLIDEAN,
+                        centroidDp
+                    )
+                );
+            }
+        }
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void scoreFromMemorySegmentBulk(Blackhole bh) throws IOException {
+        for (int j = 0; j < numQueries; j++) {
+            in.seek(0);
+            for (int i = 0; i < numVectors; i += ES91Int4VectorsScorer.BULK_SIZE) {
+                scorer.scoreBulk(
+                    binaryQueries[j],
+                    queryCorrections.lowerInterval(),
+                    queryCorrections.upperInterval(),
+                    queryCorrections.quantizedComponentSum(),
+                    queryCorrections.additionalCorrection(),
+                    VectorSimilarityFunction.EUCLIDEAN,
+                    centroidDp,
+                    scores
+                );
+                for (float score : scores) {
+                    bh.consume(score);
+                }
             }
         }
     }

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

@@ -303,7 +303,6 @@ else if (node.getSettingKeys().contains("telemetry.metrics.enabled") == false) {
         if (cliDebug) {
             enableCliDebug();
         }
-        enableEntitlements();
     }
 
     @TaskAction

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java

@@ -88,12 +88,4 @@ default void enableCliDebug() {
             }
         }
     }
-
-    default void enableEntitlements() {
-        for (ElasticsearchCluster cluster : getClusters()) {
-            for (ElasticsearchNode node : cluster.getNodes()) {
-                node.cliJvmArgs("-Des.entitlements.enabled=true");
-            }
-        }
-    }
 }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/MachineDependentHeap.java

@@ -40,6 +40,8 @@ public class MachineDependentHeap {
 
     private static final FeatureFlag NEW_ML_MEMORY_COMPUTATION_FEATURE_FLAG = new FeatureFlag("new_ml_memory_computation");
 
+    private boolean useNewMlMemoryComputation = false;
+
     public MachineDependentHeap() {}
 
     /**
@@ -55,6 +57,11 @@ public final List<String> determineHeapSettings(
         SystemMemoryInfo systemMemoryInfo,
         List<String> userDefinedJvmOptions
     ) throws IOException, InterruptedException {
+        if (userDefinedJvmOptions.contains("-Des.new_ml_memory_computation_feature_flag_enabled=true")
+            || NEW_ML_MEMORY_COMPUTATION_FEATURE_FLAG.isEnabled()) {
+            useNewMlMemoryComputation = true;
+        }
+
         // TODO: this could be more efficient, to only parse final options once
         final Map<String, JvmOption> finalJvmOptions = JvmOption.findFinalOptions(userDefinedJvmOptions);
         if (isMaxHeapSpecified(finalJvmOptions) || isMinHeapSpecified(finalJvmOptions) || isInitialHeapSpecified(finalJvmOptions)) {
@@ -107,7 +114,7 @@ protected int getHeapSizeMb(Settings nodeSettings, MachineNodeRole role, long av
             case ML_ONLY -> {
                 double heapFractionBelow16GB = 0.4;
                 double heapFractionAbove16GB = 0.1;
-                if (NEW_ML_MEMORY_COMPUTATION_FEATURE_FLAG.isEnabled()) {
+                if (useNewMlMemoryComputation) {
                     heapFractionBelow16GB = 0.4 / (1.0 + JvmErgonomics.DIRECT_MEMORY_TO_HEAP_FACTOR);
                     heapFractionAbove16GB = 0.1 / (1.0 + JvmErgonomics.DIRECT_MEMORY_TO_HEAP_FACTOR);
                 }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -11,7 +11,6 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.jdk.RuntimeVersionFeature;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -85,7 +84,6 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
             maybeSetActiveProcessorCount(nodeSettings),
             maybeSetReplayFile(distroType, isHotspot),
             maybeWorkaroundG1Bug(),
-            maybeAllowSecurityManager(useEntitlements),
             maybeAttachEntitlementAgent(esHome, useEntitlements)
         ).flatMap(s -> s).toList();
     }
@@ -160,14 +158,6 @@ private static Stream<String> maybeWorkaroundG1Bug() {
         return Stream.of();
     }
 
-    private static Stream<String> maybeAllowSecurityManager(boolean useEntitlements) {
-        if (RuntimeVersionFeature.isSecurityManagerAvailable()) {
-            // Will become conditional on useEntitlements once entitlements can run without SM
-            return Stream.of("-Djava.security.manager=allow");
-        }
-        return Stream.of();
-    }
-
     private static Stream<String> maybeAttachEntitlementAgent(Path esHome, boolean useEntitlements) {
         if (useEntitlements == false) {
             return Stream.empty();
@@ -191,7 +181,6 @@ private static Stream<String> maybeAttachEntitlementAgent(Path esHome, boolean u
         // into java.base, we must export the bridge from java.base to these modules, as a comma-separated list
         String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
         return Stream.of(
-            "-Des.entitlements.enabled=true",
             "-XX:+EnableDynamicAgentLoading",
             "-Djdk.attach.allowAttachSelf=true",
             "--patch-module=java.base=" + bridgeJar,

docs/changelog/130544.yaml

@@ -0,0 +1,6 @@
+pr: 130544
+summary: Sync Inference with Trained Model stats
+area: Machine Learning
+type: bug
+issues:
+ - 130339

docs/changelog/130947.yaml

@@ -0,0 +1,5 @@
+pr: 130947
+summary: "[main]Prepare Index Like fix for backport to 9.1 and 8.19"
+area: ES|QL
+type: bug
+issues: []

docs/changelog/131113.yaml

@@ -0,0 +1,5 @@
+pr: 131113
+summary: Including `max_tokens` through the Service API for Anthropic
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/131200.yaml

@@ -0,0 +1,5 @@
+pr: 131200
+summary: Improve lost-increment message in repo analysis
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/internal/GeneralArchitectureGuide.md

@@ -182,12 +182,12 @@ capabilities.
 
 ## Serializations
 
-## Settings
+# Settings
 
 Elasticsearch supports [cluster-level settings][] and [index-level settings][], configurable via [node-level file settings][]
 (e.g. `elasticsearch.yml` file), command line arguments and REST APIs.
 
-### Declaring a Setting
+## Declaring a Setting
 
 [cluster-level settings]: https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-update-settings.html
 [index-level settings]: https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html
@@ -222,7 +222,7 @@ settings.
 [SettingsModule constructor]: https://github.com/elastic/elasticsearch/blob/v8.13.2/server/src/main/java/org/elasticsearch/node/NodeConstruction.java#L491-L495
 [getSettings()]: https://github.com/elastic/elasticsearch/blob/v8.13.2/server/src/main/java/org/elasticsearch/plugins/Plugin.java#L203-L208
 
-### Dynamically updating a Setting
+## Dynamically updating a Setting
 
 Externally, [TransportClusterUpdateSettingsAction][] and [TransportUpdateSettingsAction][] (and the corresponding REST endpoints)
 allow users to dynamically change cluster and index settings, respectively. Internally, `AbstractScopedSettings` (parent class
@@ -244,9 +244,9 @@ state must ever be reloaded from persisted state.
 [Metadata]: https://github.com/elastic/elasticsearch/blob/v8.13.2/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java#L212-L213
 [applied here]: https://github.com/elastic/elasticsearch/blob/v8.13.2/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java#L2437
 
-## Deprecations
+# Deprecations
 
-## Backwards Compatibility
+# Backwards Compatibility
 
 major releases are mostly about breaking compatibility and dropping deprecated functionality.
 
@@ -292,18 +292,18 @@ See the [public upgrade docs][] for the upgrade process.
 
 [public upgrade docs]: https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html
 
-## Plugins
+# Plugins
 
 (what warrants a plugin?)
 
 (what plugins do we have?)
 
-## Testing
+# Testing
 
 (Overview of our testing frameworks. Discuss base test classes.)
 
-### Unit Testing
+## Unit Testing
 
-### REST Testing
+## REST Testing
 
-### Integration Testing
+## Integration Testing