Merge branch 'main' into markjhoy/add_sparse_vector_token_pruning_index_options

Author: markjhoy

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java

@@ -10,7 +10,6 @@
 package org.elasticsearch.gradle.internal;
 
 import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin;
-import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
@@ -28,14 +27,14 @@ public void apply(final Project project) {
         var depsInfo = project.getTasks().register("dependenciesInfo", DependenciesInfoTask.class);
 
         depsInfo.configure(t -> {
-            t.setRuntimeConfiguration(project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME));
-            t.setCompileOnlyConfiguration(
-                project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)
-            );
-            t.getConventionMapping().map("mappings", () -> {
-                var depLic = project.getTasks().named("dependencyLicenses", DependencyLicensesTask.class);
-                return depLic.get().getMappings();
-            });
+            var runtimeConfiguration = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
+            t.getRuntimeArtifacts().set(project.getProviders().provider(() -> runtimeConfiguration.getIncoming().getArtifacts()));
+            t.getClasspath().from(runtimeConfiguration);
+            var compileOnlyConfiguration = project.getConfigurations()
+                .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
+            t.getCompileOnlyArtifacts().set(project.getProviders().provider(() -> compileOnlyConfiguration.getIncoming().getArtifacts()));
+            t.getClasspath().from(compileOnlyConfiguration);
+
         });
         Configuration dependenciesInfoFilesConfiguration = project.getConfigurations().create("dependenciesInfoFiles");
         dependenciesInfoFilesConfiguration.setCanBeResolved(false);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java

@@ -11,19 +11,22 @@
 
 import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask;
 import org.elasticsearch.gradle.internal.precommit.LicenseAnalyzer;
-import org.gradle.api.artifacts.Configuration;
-import org.gradle.api.artifacts.Dependency;
-import org.gradle.api.artifacts.DependencySet;
-import org.gradle.api.artifacts.ModuleVersionIdentifier;
+import org.gradle.api.artifacts.ArtifactCollection;
 import org.gradle.api.artifacts.ProjectDependency;
+import org.gradle.api.artifacts.component.ModuleComponentIdentifier;
+import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.DirectoryProperty;
 import org.gradle.api.file.ProjectLayout;
 import org.gradle.api.internal.ConventionTask;
 import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.MapProperty;
+import org.gradle.api.provider.Property;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputDirectory;
 import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.OutputFile;
 import org.gradle.api.tasks.TaskAction;
@@ -34,6 +37,7 @@
 import java.nio.file.StandardOpenOption;
 import java.util.Arrays;
 import java.util.LinkedHashMap;
+import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -51,31 +55,58 @@
  *     <li>license: <a href="https://spdx.org/licenses/">SPDX license</a> identifier, custom license or UNKNOWN.</li>
  * </ul>
  */
-public class DependenciesInfoTask extends ConventionTask {
+public abstract class DependenciesInfoTask extends ConventionTask {
 
-    private final DirectoryProperty licensesDir;
+    @Inject
+    public abstract ProviderFactory getProviderFactory();
 
-    @OutputFile
-    private File outputFile;
+    /**
+     * We have to use ArtifactCollection instead of ResolvedArtifactResult here as we're running
+     * into a an issue in Gradle: https://github.com/gradle/gradle/issues/27582
+     */
 
-    private LinkedHashMap<String, String> mappings;
+    @Internal
+    abstract Property<ArtifactCollection> getRuntimeArtifacts();
 
-    public Configuration getRuntimeConfiguration() {
-        return runtimeConfiguration;
+    @Input
+    public Provider<Set<ModuleComponentIdentifier>> getRuntimeModules() {
+        return mapToModuleComponentIdentifiers(getRuntimeArtifacts().get());
     }
 
-    public void setRuntimeConfiguration(Configuration runtimeConfiguration) {
-        this.runtimeConfiguration = runtimeConfiguration;
-    }
+    @Internal
+    abstract Property<ArtifactCollection> getCompileOnlyArtifacts();
 
-    public Configuration getCompileOnlyConfiguration() {
-        return compileOnlyConfiguration;
+    @Input
+    public Provider<Set<ModuleComponentIdentifier>> getCompileOnlyModules() {
+        return mapToModuleComponentIdentifiers(getCompileOnlyArtifacts().get());
     }
 
-    public void setCompileOnlyConfiguration(Configuration compileOnlyConfiguration) {
-        this.compileOnlyConfiguration = compileOnlyConfiguration;
+    /**
+     * We need to track file inputs here from the configurations we inspect to ensure we dont miss any
+     * artifact transforms that might be applied and fail due to missing task dependency to jar
+     * generating tasks.
+     * */
+    @InputFiles
+    abstract ConfigurableFileCollection getClasspath();
+
+    private Provider<Set<ModuleComponentIdentifier>> mapToModuleComponentIdentifiers(ArtifactCollection artifacts) {
+        return getProviderFactory().provider(
+            () -> artifacts.getArtifacts()
+                .stream()
+                .map(r -> r.getId())
+                .filter(id -> id instanceof ModuleComponentIdentifier)
+                .map(id -> (ModuleComponentIdentifier) id)
+                .collect(Collectors.toSet())
+        );
     }
 
+    private final DirectoryProperty licensesDir;
+
+    @OutputFile
+    private File outputFile;
+
+    private LinkedHashMap<String, String> mappings;
+
     /**
      * Directory to read license files
      */
@@ -102,17 +133,6 @@ public void setOutputFile(File outputFile) {
         this.outputFile = outputFile;
     }
 
-    /**
-     * Dependencies to gather information from.
-     */
-    @InputFiles
-    private Configuration runtimeConfiguration;
-    /**
-     * We subtract compile-only dependencies.
-     */
-    @InputFiles
-    private Configuration compileOnlyConfiguration;
-
     @Inject
     public DependenciesInfoTask(ProjectLayout projectLayout, ObjectFactory objectFactory, ProviderFactory providerFactory) {
         this.licensesDir = objectFactory.directoryProperty();
@@ -123,22 +143,18 @@ public DependenciesInfoTask(ProjectLayout projectLayout, ObjectFactory objectFac
 
     @TaskAction
     public void generateDependenciesInfo() throws IOException {
-        final DependencySet runtimeDependencies = runtimeConfiguration.getAllDependencies();
-        // we have to resolve the transitive dependencies and create a group:artifactId:version map
-
-        final Set<String> compileOnlyArtifacts = compileOnlyConfiguration.getResolvedConfiguration()
-            .getResolvedArtifacts()
-            .stream()
-            .map(r -> {
-                ModuleVersionIdentifier id = r.getModuleVersion().getId();
-                return id.getGroup() + ":" + id.getName() + ":" + id.getVersion();
-            })
-            .collect(Collectors.toSet());
 
+        final Set<String> compileOnlyIds = getCompileOnlyModules().map(
+            set -> set.stream()
+                .map(id -> id.getModuleIdentifier().getGroup() + ":" + id.getModuleIdentifier().getName() + ":" + id.getVersion())
+                .collect(Collectors.toSet())
+        ).get();
         final StringBuilder output = new StringBuilder();
-        for (final Dependency dep : runtimeDependencies) {
+        Map<String, String> mappings = getMappings().get();
+        for (final ModuleComponentIdentifier dep : getRuntimeModules().get()) {
             // we do not need compile-only dependencies here
-            if (compileOnlyArtifacts.contains(dep.getGroup() + ":" + dep.getName() + ":" + dep.getVersion())) {
+            String moduleName = dep.getModuleIdentifier().getName();
+            if (compileOnlyIds.contains(dep.getGroup() + ":" + moduleName + ":" + dep.getVersion())) {
                 continue;
             }
 
@@ -147,25 +163,20 @@ public void generateDependenciesInfo() throws IOException {
                 continue;
             }
 
-            final String url = createURL(dep.getGroup(), dep.getName(), dep.getVersion());
-            final String dependencyName = DependencyLicensesTask.getDependencyName(getMappings(), dep.getName());
-            getLogger().info("mapped dependency " + dep.getGroup() + ":" + dep.getName() + " to " + dependencyName + " for license info");
+            final String url = createURL(dep.getGroup(), moduleName, dep.getVersion());
+            final String dependencyName = DependencyLicensesTask.getDependencyName(mappings, moduleName);
+            getLogger().info("mapped dependency " + dep.getGroup() + ":" + moduleName + " to " + dependencyName + " for license info");
 
             final String licenseType = getLicenseType(dep.getGroup(), dependencyName);
-            output.append(dep.getGroup() + ":" + dep.getName() + "," + dep.getVersion() + "," + url + "," + licenseType + "\n");
+            output.append(dep.getGroup() + ":" + moduleName + "," + dep.getVersion() + "," + url + "," + licenseType + "\n");
         }
 
         Files.writeString(outputFile.toPath(), output.toString(), StandardOpenOption.CREATE);
     }
 
     @Input
-    public LinkedHashMap<String, String> getMappings() {
-        return mappings;
-    }
-
-    public void setMappings(LinkedHashMap<String, String> mappings) {
-        this.mappings = mappings;
-    }
+    @Optional
+    public abstract MapProperty<String, String> getMappings();
 
     /**
      * Create an URL on <a href="https://repo1.maven.org/maven2/">Maven Central</a>

muted-tests.yml

@@ -559,27 +559,27 @@ tests:
 - class: org.elasticsearch.test.apmintegration.TracesApmIT
   method: testApmIntegration
   issue: https://github.com/elastic/elasticsearch/issues/129651
-- class: org.elasticsearch.search.query.RescoreKnnVectorQueryIT
-  method: testKnnSearchRescore
-  issue: https://github.com/elastic/elasticsearch/issues/129713
 - class: org.elasticsearch.snapshots.SnapshotShutdownIT
   method: testSnapshotShutdownProgressTracker
   issue: https://github.com/elastic/elasticsearch/issues/129752
 - class: org.elasticsearch.xpack.security.SecurityRolesMultiProjectIT
   method: testUpdatingFileBasedRoleAffectsAllProjects
   issue: https://github.com/elastic/elasticsearch/issues/129775
-- class: org.elasticsearch.search.query.RescoreKnnVectorQueryIT
-  method: testKnnQueryRescore
-  issue: https://github.com/elastic/elasticsearch/issues/129809
-- class: org.elasticsearch.search.query.RescoreKnnVectorQueryIT
-  method: testKnnRetriever
-  issue: https://github.com/elastic/elasticsearch/issues/129818
+- class: org.elasticsearch.qa.verify_version_constants.VerifyVersionConstantsIT
+  method: testLuceneVersionConstant
+  issue: https://github.com/elastic/elasticsearch/issues/125638
+- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
+  method: test
+  issue: https://github.com/elastic/elasticsearch/issues/129819
 - class: org.elasticsearch.index.engine.ThreadPoolMergeExecutorServiceDiskSpaceTests
   method: testAbortingOrRunningMergeTaskHoldsUpBudget
   issue: https://github.com/elastic/elasticsearch/issues/129823
 - class: org.elasticsearch.index.store.FsDirectoryFactoryTests
   method: testPreload
   issue: https://github.com/elastic/elasticsearch/issues/129852
+- class: org.elasticsearch.xpack.rank.rrf.RRFRankClientYamlTestSuiteIT
+  method: test {yaml=rrf/950_pinned_interaction/rrf with pinned retriever as a sub-retriever}
+  issue: https://github.com/elastic/elasticsearch/issues/129845
 
 # Examples:
 #

server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java

@@ -85,7 +85,7 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
     public void setup() throws IOException {
         String type = randomFrom(
             Arrays.stream(VectorIndexType.values())
-                .filter(VectorIndexType::isQuantized)
+                .filter(t -> t.isQuantized() && t.isEnabled())
                 .map(t -> t.name().toLowerCase(Locale.ROOT))
                 .collect(Collectors.toCollection(ArrayList::new))
         );

server/src/main/java/org/elasticsearch/cluster/metadata/StreamsMetadata.java

@@ -59,7 +59,13 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.STREAMS_LOGS_SUPPORT;
+        return TransportVersions.STREAMS_LOGS_SUPPORT_8_19;
+    }
+
+    @Override
+    public boolean supportsVersion(TransportVersion version) {
+        return version.onOrAfter(TransportVersions.STREAMS_LOGS_SUPPORT)
+            || version.isPatchFrom(TransportVersions.STREAMS_LOGS_SUPPORT_8_19);
     }
 
     public static NamedDiff<Metadata.ProjectCustom> readDiffFrom(StreamInput in) throws IOException {

server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java

@@ -1655,11 +1655,16 @@ public boolean supportsElementType(ElementType elementType) {
             public boolean supportsDimension(int dims) {
                 return true;
             }
+
+            @Override
+            public boolean isEnabled() {
+                return IVF_FORMAT.isEnabled();
+            }
         };
 
         public static Optional<VectorIndexType> fromString(String type) {
             return Stream.of(VectorIndexType.values())
-                .filter(vectorIndexType -> vectorIndexType != VectorIndexType.BBQ_IVF || IVF_FORMAT.isEnabled())
+                .filter(VectorIndexType::isEnabled)
                 .filter(vectorIndexType -> vectorIndexType.name.equals(type))
                 .findFirst();
         }
@@ -1686,6 +1691,10 @@ public boolean isQuantized() {
             return quantized;
         }
 
+        public boolean isEnabled() {
+            return true;
+        }
+
         public String getName() {
             return name;
         }

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java

@@ -20,7 +20,6 @@ public enum FeatureFlag {
     SUB_OBJECTS_AUTO_ENABLED("es.sub_objects_auto_feature_flag_enabled=true", Version.fromString("8.16.0"), null),
     DOC_VALUES_SKIPPER("es.doc_values_skipper_feature_flag_enabled=true", Version.fromString("8.18.1"), null),
     USE_LUCENE101_POSTINGS_FORMAT("es.use_lucene101_postings_format_feature_flag_enabled=true", Version.fromString("9.1.0"), null),
-    INFERENCE_CUSTOM_SERVICE_ENABLED("es.inference_custom_service_feature_flag_enabled=true", Version.fromString("8.19.0"), null),
     IVF_FORMAT("es.ivf_format_feature_flag_enabled=true", Version.fromString("9.1.0"), null),
     LOGS_STREAM("es.logs_stream_feature_flag_enabled=true", Version.fromString("9.1.0"), null);
 

x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/BaseMockEISAuthServerTest.java

@@ -14,7 +14,6 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
-import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.junit.ClassRule;
@@ -45,7 +44,6 @@ public class BaseMockEISAuthServerTest extends ESRestTestCase {
         // This plugin is located in the inference/qa/test-service-plugin package, look for TestInferenceServicePlugin
         .plugin("inference-service-test")
         .user("x_pack_rest_user", "x-pack-test-password")
-        .feature(FeatureFlag.INFERENCE_CUSTOM_SERVICE_ENABLED)
         .build();
 
     // The reason we're doing this is to make sure the mock server is initialized first so we can get the address before communicating

x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java

@@ -20,7 +20,6 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
-import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -51,7 +50,6 @@ public class InferenceBaseRestTest extends ESRestTestCase {
         .setting("xpack.security.enabled", "true")
         .plugin("inference-service-test")
         .user("x_pack_rest_user", "x-pack-test-password")
-        .feature(FeatureFlag.INFERENCE_CUSTOM_SERVICE_ENABLED)
         .build();
 
     @ClassRule

x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceGetModelsWithElasticInferenceServiceIT.java

@@ -33,7 +33,7 @@ public void testGetDefaultEndpoints() throws IOException {
         var allModels = getAllModels();
         var chatCompletionModels = getModels("_all", TaskType.CHAT_COMPLETION);
 
-        assertThat(allModels, hasSize(5));
+        assertThat(allModels, hasSize(6));
         assertThat(chatCompletionModels, hasSize(1));
 
         for (var model : chatCompletionModels) {
@@ -42,6 +42,7 @@ public void testGetDefaultEndpoints() throws IOException {
 
         assertInferenceIdTaskType(allModels, ".rainbow-sprinkles-elastic", TaskType.CHAT_COMPLETION);
         assertInferenceIdTaskType(allModels, ".elser-v2-elastic", TaskType.SPARSE_EMBEDDING);
+        assertInferenceIdTaskType(allModels, ".rerank-v1-elastic", TaskType.RERANK);
     }
 
     private static void assertInferenceIdTaskType(List<Map<String, Object>> models, String inferenceId, TaskType taskType) {