move GPU plugin to x-pack

Author: ChrisHegarty

server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java

@@ -17,7 +17,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
-import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.VectorIndexType;
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.BasicVectorIndexType;
 import org.elasticsearch.index.mapper.vectors.DenseVectorScriptDocValues;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -84,8 +84,8 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
     @Before
     public void setup() throws IOException {
         String type = randomFrom(
-            Arrays.stream(VectorIndexType.values())
-                .filter(VectorIndexType::isQuantized)
+            Arrays.stream(BasicVectorIndexType.values())
+                .filter(BasicVectorIndexType::isQuantized)
                 .map(t -> t.name().toLowerCase(Locale.ROOT))
                 .collect(Collectors.toCollection(ArrayList::new))
         );

server/src/main/java/module-info.java

@@ -457,8 +457,7 @@
             org.elasticsearch.index.codec.vectors.es816.ES816HnswBinaryQuantizedVectorsFormat,
             org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsFormat,
             org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormat,
-            org.elasticsearch.index.codec.vectors.IVFVectorsFormat,
-            org.elasticsearch.index.codec.vectors.GPUVectorsFormat;
+            org.elasticsearch.index.codec.vectors.IVFVectorsFormat;
 
     provides org.apache.lucene.codecs.Codec
         with

server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

@@ -804,7 +804,8 @@ private static void postProcessDynamicArrayMapping(DocumentParserContext context
 
             DenseVectorFieldMapper.Builder builder = new DenseVectorFieldMapper.Builder(
                 fieldName,
-                context.indexSettings().getIndexVersionCreated()
+                context.indexSettings().getIndexVersionCreated(),
+                context::denseVectorIndexType
             );
             DenseVectorFieldMapper denseVectorFieldMapper = builder.build(builderContext);
             context.updateDynamicMappers(fullFieldName, List.of(denseVectorFieldMapper));

server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.xcontent.FilterXContentParserWrapper;
 import org.elasticsearch.xcontent.FlatteningXContentParser;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -287,6 +288,10 @@ public final RootObjectMapper root() {
         return this.mappingLookup.getMapping().getRoot();
     }
 
+    public final DenseVectorFieldMapper.VectorIndexType denseVectorIndexType(String type) {
+        return mappingParserContext.denseVectorIndexType(type);
+    }
+
     public final ObjectMapper parent() {
         return parent;
     }

server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -131,6 +131,18 @@ default boolean supportsVersion(IndexVersion indexCreatedVersion) {
         }
     }
 
+    // public interface DenseVectorTypeParser {
+    // DenseVectorFieldMapper parse(String name, Map<String, Object> node, MappingParserContext parserContext)
+    // throws MapperParsingException;
+    //
+    // /**
+    // * Whether we can parse this type on indices with the given index created version.
+    // */
+    // default boolean supportsVersion(IndexVersion indexCreatedVersion) {
+    // return indexCreatedVersion.onOrAfter(IndexVersions.MINIMUM_READONLY_COMPATIBLE);
+    // }
+    // }
+
     private final String leafName;
 
     @SuppressWarnings("this-escape")

server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.plugins.FieldPredicate;
 import org.elasticsearch.plugins.MapperPlugin;
 
@@ -31,12 +32,14 @@ public final class MapperRegistry {
     private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers6x;
     private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers5x;
     private final Function<String, FieldPredicate> fieldFilter;
+    private final Map<String, DenseVectorFieldMapper.VectorIndexType> denseVectorIndexTypes;
 
     public MapperRegistry(
         Map<String, Mapper.TypeParser> mapperParsers,
         Map<String, RuntimeField.Parser> runtimeFieldParsers,
         Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers,
-        Function<String, FieldPredicate> fieldFilter
+        Function<String, FieldPredicate> fieldFilter,
+        Map<String, DenseVectorFieldMapper.VectorIndexType> denseVectorIndexTypes
     ) {
         this.mapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(mapperParsers));
         this.runtimeFieldParsers = runtimeFieldParsers;
@@ -50,6 +53,7 @@ public MapperRegistry(
         metadata5x.put(LegacyTypeFieldMapper.NAME, LegacyTypeFieldMapper.PARSER);
         this.metadataMapperParsers5x = metadata5x;
         this.fieldFilter = fieldFilter;
+        this.denseVectorIndexTypes = Collections.unmodifiableMap(denseVectorIndexTypes);
     }
 
     /**
@@ -100,4 +104,11 @@ public Map<String, MetadataFieldMapper.TypeParser> getMetadataMapperParsers(Inde
     public Function<String, FieldPredicate> getFieldFilter() {
         return fieldFilter;
     }
+
+    /**
+     * Returns ...
+     */
+    public DenseVectorFieldMapper.VectorIndexType getDenseVectorIndexType(String type) {
+        return denseVectorIndexTypes.get(type);
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -238,6 +238,7 @@ public MapperService(
             similarityService::getSimilarity,
             type -> mapperRegistry.getMapperParser(type, indexVersionCreated),
             mapperRegistry.getRuntimeFieldParsers()::get,
+            mapperRegistry::getDenseVectorIndexType,
             indexVersionCreated,
             clusterTransportVersion,
             searchExecutionContextSupplier,

server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.similarity.SimilarityProvider;
 import org.elasticsearch.script.ScriptCompiler;
@@ -33,6 +34,7 @@ public class MappingParserContext {
     private final Function<String, SimilarityProvider> similarityLookupService;
     private final Function<String, Mapper.TypeParser> typeParsers;
     private final Function<String, RuntimeField.Parser> runtimeFieldParsers;
+    private final Function<String, DenseVectorFieldMapper.VectorIndexType> denseVectorIndexType;
     private final IndexVersion indexVersionCreated;
     private final Supplier<TransportVersion> clusterTransportVersion;
     private final Supplier<SearchExecutionContext> searchExecutionContextSupplier;
@@ -48,6 +50,7 @@ public MappingParserContext(
         Function<String, SimilarityProvider> similarityLookupService,
         Function<String, Mapper.TypeParser> typeParsers,
         Function<String, RuntimeField.Parser> runtimeFieldParsers,
+        Function<String, DenseVectorFieldMapper.VectorIndexType> denseVectorIndexType,
         IndexVersion indexVersionCreated,
         Supplier<TransportVersion> clusterTransportVersion,
         Supplier<SearchExecutionContext> searchExecutionContextSupplier,
@@ -60,6 +63,7 @@ public MappingParserContext(
         this.similarityLookupService = similarityLookupService;
         this.typeParsers = typeParsers;
         this.runtimeFieldParsers = runtimeFieldParsers;
+        this.denseVectorIndexType = denseVectorIndexType;
         this.indexVersionCreated = indexVersionCreated;
         this.clusterTransportVersion = clusterTransportVersion;
         this.searchExecutionContextSupplier = searchExecutionContextSupplier;
@@ -99,6 +103,10 @@ public RuntimeField.Parser runtimeFieldParser(String type) {
         return runtimeFieldParsers.apply(type);
     }
 
+    public DenseVectorFieldMapper.VectorIndexType denseVectorIndexType(String type) {
+        return denseVectorIndexType.apply(type);
+    }
+
     public IndexVersion indexVersionCreated() {
         return indexVersionCreated;
     }
@@ -163,6 +171,7 @@ private static class MultiFieldParserContext extends MappingParserContext {
                 in.similarityLookupService,
                 in.typeParsers,
                 in.runtimeFieldParsers,
+                null, // TODO: serialization
                 in.indexVersionCreated,
                 in.clusterTransportVersion,
                 in.searchExecutionContextSupplier,
@@ -193,6 +202,7 @@ private static class DynamicTemplateParserContext extends MappingParserContext {
                 in.similarityLookupService,
                 in.typeParsers,
                 in.runtimeFieldParsers,
+                null, // TODO: serialization
                 in.indexVersionCreated,
                 in.clusterTransportVersion,
                 in.searchExecutionContextSupplier,