Merge branch 'main' into resharding_search_filters_2

Author: lkts

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OptimizedScalarQuantizerBenchmark.java

@@ -42,6 +42,7 @@ public class OptimizedScalarQuantizerBenchmark {
     int dims;
 
     float[] vector;
+    float[] scratch;
     float[] centroid;
     int[] destination;
 
@@ -57,6 +58,7 @@ public void init() {
         destination = new int[dims];
         vector = new float[dims];
         centroid = new float[dims];
+        scratch = new float[dims];
         for (int i = 0; i < dims; ++i) {
             vector[i] = random.nextFloat();
             centroid[i] = random.nextFloat();
@@ -65,14 +67,14 @@ public void init() {
 
     @Benchmark
     public int[] scalar() {
-        osq.scalarQuantize(vector, destination, bits, centroid);
+        osq.scalarQuantize(vector, scratch, destination, bits, centroid);
         return destination;
     }
 
     @Benchmark
     @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
     public int[] vector() {
-        osq.scalarQuantize(vector, destination, bits, centroid);
+        osq.scalarQuantize(vector, scratch, destination, bits, centroid);
         return destination;
     }
 }

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesBuildService.java

@@ -22,7 +22,7 @@
 import java.util.Properties;
 import javax.inject.Inject;
 
-abstract class VersionPropertiesBuildService implements BuildService<VersionPropertiesBuildService.Params>, AutoCloseable {
+public abstract class VersionPropertiesBuildService implements BuildService<VersionPropertiesBuildService.Params>, AutoCloseable {
 
     private final Properties properties;
 

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy

@@ -443,4 +443,46 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
         assertUpperBound("9.2", "new_tv,8124000")
         assertReferableDefinition("new_tv", "8124000")
     }
+
+    def "alternate upper bound larger"() {
+        given:
+        referencedTransportVersion("new_tv")
+        file("myserver/alt_upper_bound.csv").text = "some_tv,8126000"
+        file("myserver/build.gradle") << """
+            tasks.named('generateTransportVersionDefinition') {
+                alternateUpperBoundFile = project.file("alt_upper_bound.csv")
+            }
+            tasks.named('validateTransportVersionResources') {
+                shouldValidateDensity = false
+            }
+        """
+
+        when:
+        def result = runGenerateAndValidateTask().build()
+        then:
+        assertGenerateAndValidateSuccess(result)
+        assertUpperBound("9.2", "new_tv,8127000")
+        assertReferableDefinition("new_tv", "8127000")
+    }
+
+    def "alternate upper bound less"() {
+        given:
+        referencedTransportVersion("new_tv")
+        file("myserver/alt_upper_bound.csv").text = "some_tv,8122100"
+        file("myserver/build.gradle") << """
+            tasks.named('generateTransportVersionDefinition') {
+                alternateUpperBoundFile = project.file("alt_upper_bound.csv")
+            }
+            tasks.named('validateTransportVersionResources') {
+                shouldValidateDensity = false
+            }
+        """
+
+        when:
+        def result = runGenerateAndValidateTask().build()
+        then:
+        assertGenerateAndValidateSuccess(result)
+        assertUpperBound("9.2", "new_tv,8124000")
+        assertReferableDefinition("new_tv", "8124000")
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java

@@ -11,9 +11,11 @@
 
 import org.gradle.api.DefaultTask;
 import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.RegularFileProperty;
 import org.gradle.api.provider.Property;
 import org.gradle.api.services.ServiceReference;
 import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.InputFile;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.PathSensitive;
@@ -22,6 +24,9 @@
 import org.gradle.api.tasks.options.Option;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -74,6 +79,14 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask
     @Input
     public abstract Property<String> getCurrentUpperBoundName();
 
+    /**
+     * An additional upper bound file that will be consulted when generating a transport version.
+     * The larger of this and the current upper bound will be used to create the new primary id.
+     */
+    @InputFile
+    @Optional
+    public abstract RegularFileProperty getAlternateUpperBoundFile();
+
     @TaskAction
     public void run() throws IOException {
         TransportVersionResourcesService resources = getResourceService().get();
@@ -119,7 +132,7 @@ private List<TransportVersionId> updateUpperBounds(
                 if (targetId == null) {
                     // Case: an id doesn't yet exist for this upper bound, so create one
                     int targetIncrement = upperBoundName.equals(currentUpperBoundName) ? increment : 1;
-                    targetId = TransportVersionId.fromInt(existingUpperBound.definitionId().complete() + targetIncrement);
+                    targetId = createTargetId(existingUpperBound, targetIncrement);
                     var newUpperBound = new TransportVersionUpperBound(upperBoundName, definitionName, targetId);
                     resources.writeUpperBound(newUpperBound);
                 }
@@ -237,4 +250,21 @@ private TransportVersionId maybeGetExistingId(
         return null; // no existing id for this upper bound
     }
 
+    private TransportVersionId createTargetId(TransportVersionUpperBound existingUpperBound, int increment) throws IOException {
+        int currentId = existingUpperBound.definitionId().complete();
+
+        // allow for an alternate upper bound file to be consulted. This supports Serverless basing its
+        // own transport version ids on the greater of server or serverless
+        if (getAlternateUpperBoundFile().isPresent()) {
+            Path altUpperBoundPath = getAlternateUpperBoundFile().get().getAsFile().toPath();
+            String contents = Files.readString(altUpperBoundPath, StandardCharsets.UTF_8);
+            var altUpperBound = TransportVersionUpperBound.fromString(altUpperBoundPath, contents);
+            if (altUpperBound.definitionId().complete() > currentId) {
+                currentId = altUpperBound.definitionId().complete();
+            }
+        }
+
+        return TransportVersionId.fromInt(currentId + increment);
+    }
+
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionReferencesPlugin.java

@@ -10,6 +10,8 @@
 package org.elasticsearch.gradle.internal.transport;
 
 import org.elasticsearch.gradle.internal.ProjectSubscribeServicePlugin;
+import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin;
+import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -23,6 +25,7 @@ public class TransportVersionReferencesPlugin implements Plugin<Project> {
     @Override
     public void apply(Project project) {
         project.getPluginManager().apply(LifecycleBasePlugin.class);
+        project.getPluginManager().apply(PrecommitTaskPlugin.class);
 
         project.getPlugins()
             .apply(ProjectSubscribeServicePlugin.class)
@@ -50,6 +53,6 @@ public void apply(Project project) {
                 t.setDescription("Validates that all TransportVersion references used in the project have an associated definition file");
                 t.getReferencesFile().set(collectTask.get().getOutputFile());
             });
-        project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask));
+        project.getTasks().named(PrecommitPlugin.PRECOMMIT_TASK_NAME).configure(t -> t.dependsOn(validateTask));
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java

@@ -12,6 +12,8 @@
 import org.elasticsearch.gradle.Version;
 import org.elasticsearch.gradle.internal.ProjectSubscribeServicePlugin;
 import org.elasticsearch.gradle.internal.conventions.VersionPropertiesPlugin;
+import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin;
+import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.file.Directory;
@@ -30,6 +32,7 @@ public class TransportVersionResourcesPlugin implements Plugin<Project> {
     public void apply(Project project) {
         project.getPluginManager().apply(LifecycleBasePlugin.class);
         project.getPluginManager().apply(VersionPropertiesPlugin.class);
+        project.getPluginManager().apply(PrecommitTaskPlugin.class);
         var psService = project.getPlugins().apply(ProjectSubscribeServicePlugin.class).getService();
 
         Properties versions = (Properties) project.getExtensions().getByName(VersionPropertiesPlugin.VERSIONS_EXT);
@@ -67,7 +70,7 @@ public void apply(Project project) {
                 t.getShouldValidateDensity().convention(true);
                 t.getShouldValidatePrimaryIdNotPatch().convention(true);
             });
-        project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask));
+        project.getTasks().named(PrecommitPlugin.PRECOMMIT_TASK_NAME).configure(t -> t.dependsOn(validateTask));
 
         var generateManifestTask = project.getTasks()
             .register("generateTransportVersionManifest", GenerateTransportVersionManifestTask.class, t -> {

docs/changelog/133446.yaml

@@ -0,0 +1,5 @@
+pr: 133720
+summary: Support querying multiple indices with the simplified linear retriever
+area: Relevance
+type: enhancement
+issues: []

docs/changelog/133720.yaml

@@ -0,0 +1,6 @@
+pr: 133860
+summary: Cache Inference Endpoints
+area: Machine Learning
+type: enhancement
+issues:
+ - 133135

docs/changelog/133860.yaml

@@ -0,0 +1,6 @@
+pr: 134134
+summary: Prevent field caps from failing due to can match failure
+area: Search
+type: bug
+issues:
+ - 116106