From b5decc4c8a70c248116d8c1630852ab54a8ab514 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:40:12 -0500 Subject: [PATCH 01/14] Moving rank_vectors to a plugin and removing feature flag --- .../org.elasticsearch.script.score.txt | 2 -- .../vectors/DenseVectorFieldMapper.java | 36 +++++++++---------- .../elasticsearch/indices/IndicesModule.java | 4 --- .../action/search/SearchCapabilities.java | 12 ------- .../aggregations/AggregatorTestCase.java | 2 -- x-pack/plugin/rank-vectors/build.gradle | 26 ++++++++++++++ .../src/main/java/module-info.java | 23 ++++++++++++ .../rank/vectors/RankVectorsFeatures.java | 23 ++++++++++++ .../xpack/rank/vectors/RankVectorsPlugin.java | 32 +++++++++++++++++ .../mapper}/RankVectorsDVLeafFieldData.java | 11 +++--- .../mapper}/RankVectorsFieldMapper.java | 36 +++++++++++-------- .../mapper}/RankVectorsIndexFieldData.java | 20 ++++------- .../script/RankVectorsPainlessExtension.java | 26 ++++++++++++++ .../script/RankVectorsScoreScriptUtils.java | 13 ++++++- ...lasticsearch.features.FeatureSpecification | 8 +++++ ...asticsearch.painless.spi.PainlessExtension | 1 + .../xpack/rank/vectors/script/whitelist.txt | 13 +++++++ .../mapper}/RankVectorsFieldMapperTests.java | 16 +++------ .../mapper}/RankVectorsFieldTypeTests.java | 21 ++++------- .../RankVectorsScriptDocValuesTests.java | 17 +++------ .../RankVectorsScoreScriptUtilsTests.java | 29 +++++---------- .../vectors/script}/RankVectorsTests.java | 21 +++++------ .../test/rank_vectors/rank_vectors.yml | 8 ++--- .../rank_vectors_dv_fields_api.yml | 8 ++--- .../rank_vectors/rank_vectors_max_sim.yml | 8 ++--- 25 files changed, 254 insertions(+), 162 deletions(-) create mode 100644 x-pack/plugin/rank-vectors/build.gradle create mode 100644 x-pack/plugin/rank-vectors/src/main/java/module-info.java create mode 100644 x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsFeatures.java create mode 100644 x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java rename {server/src/main/java/org/elasticsearch/index/mapper/vectors => x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper}/RankVectorsDVLeafFieldData.java (84%) rename {server/src/main/java/org/elasticsearch/index/mapper/vectors => x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper}/RankVectorsFieldMapper.java (92%) rename {server/src/main/java/org/elasticsearch/index/mapper/vectors => x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper}/RankVectorsIndexFieldData.java (81%) create mode 100644 x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java rename {server/src/main/java/org/elasticsearch => x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors}/script/RankVectorsScoreScriptUtils.java (96%) create mode 100644 x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification create mode 100644 x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension create mode 100644 x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/whitelist.txt rename {server/src/test/java/org/elasticsearch/index/mapper/vectors => x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper}/RankVectorsFieldMapperTests.java (97%) rename {server/src/test/java/org/elasticsearch/index/mapper/vectors => x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper}/RankVectorsFieldTypeTests.java (80%) rename {server/src/test/java/org/elasticsearch/index/mapper/vectors => x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper}/RankVectorsScriptDocValuesTests.java (95%) rename {server/src/test/java/org/elasticsearch => x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors}/script/RankVectorsScoreScriptUtilsTests.java (93%) rename {server/src/test/java/org/elasticsearch/script/field/vectors => x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script}/RankVectorsTests.java (80%) rename rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_rank_vectors.yml => x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml (92%) rename modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_rank_vectors_dv_fields_api.yml => x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml (94%) rename modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_rank_vectors_max_sim.yml => x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_max_sim.yml (95%) diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt index a5118db4876cb..e76db7cfb1d26 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt @@ -50,7 +50,5 @@ static_import { double cosineSimilarity(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$CosineSimilarity double dotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$DotProduct double hamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$Hamming - double maxSimDotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.RankVectorsScoreScriptUtils$MaxSimDotProduct - double maxSimInvHamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.RankVectorsScoreScriptUtils$MaxSimInvHamming } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 8c6e874ff577f..a368329eaea50 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -102,7 +102,7 @@ public class DenseVectorFieldMapper extends FieldMapper { public static final String COSINE_MAGNITUDE_FIELD_SUFFIX = "._magnitude"; private static final float EPS = 1e-3f; - static final int BBQ_MIN_DIMS = 64; + public static final int BBQ_MIN_DIMS = 64; public static boolean isNotUnitVector(float magnitude) { return Math.abs(magnitude - 1.0f) > EPS; @@ -486,7 +486,7 @@ private VectorData parseHexEncodedVector( } @Override - VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) + public VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) throws IOException { XContentParser.Token token = context.parser().currentToken(); return switch (token) { @@ -517,17 +517,17 @@ public void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFie } @Override - int getNumBytes(int dimensions) { + public int getNumBytes(int dimensions) { return dimensions; } @Override - ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { + public ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { return ByteBuffer.wrap(new byte[numBytes]); } @Override - int parseDimensionCount(DocumentParserContext context) throws IOException { + public int parseDimensionCount(DocumentParserContext context) throws IOException { XContentParser.Token currentToken = context.parser().currentToken(); return switch (currentToken) { case START_ARRAY -> { @@ -691,7 +691,7 @@ && isNotUnitVector(squaredMagnitude)) { } @Override - VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) + public VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) throws IOException { int index = 0; float squaredMagnitude = 0; @@ -711,12 +711,12 @@ VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanCon } @Override - int getNumBytes(int dimensions) { + public int getNumBytes(int dimensions) { return dimensions * Float.BYTES; } @Override - ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { + public ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { return indexVersion.onOrAfter(LITTLE_ENDIAN_FLOAT_STORED_INDEX_VERSION) ? ByteBuffer.wrap(new byte[numBytes]).order(ByteOrder.LITTLE_ENDIAN) : ByteBuffer.wrap(new byte[numBytes]); @@ -889,7 +889,7 @@ private VectorData parseHexEncodedVector(DocumentParserContext context, IntBoole } @Override - VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) + public VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) throws IOException { XContentParser.Token token = context.parser().currentToken(); return switch (token) { @@ -920,18 +920,18 @@ public void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFie } @Override - int getNumBytes(int dimensions) { + public int getNumBytes(int dimensions) { assert dimensions % Byte.SIZE == 0; return dimensions / Byte.SIZE; } @Override - ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { + public ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { return ByteBuffer.wrap(new byte[numBytes]); } @Override - int parseDimensionCount(DocumentParserContext context) throws IOException { + public int parseDimensionCount(DocumentParserContext context) throws IOException { XContentParser.Token currentToken = context.parser().currentToken(); return switch (currentToken) { case START_ARRAY -> { @@ -974,16 +974,16 @@ public void checkDimensions(Integer dvDims, int qvDims) { abstract void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException; - abstract VectorData parseKnnVector( + public abstract VectorData parseKnnVector( DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity ) throws IOException; - abstract int getNumBytes(int dimensions); + public abstract int getNumBytes(int dimensions); - abstract ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes); + public abstract ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes); public abstract void checkVectorBounds(float[] vector); @@ -1001,7 +1001,7 @@ public void checkDimensions(Integer dvDims, int qvDims) { } } - int parseDimensionCount(DocumentParserContext context) throws IOException { + public int parseDimensionCount(DocumentParserContext context) throws IOException { int index = 0; for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) { index++; @@ -1088,7 +1088,7 @@ public static ElementType fromString(String name) { } } - static final Map namesToElementType = Map.of( + public static final Map namesToElementType = Map.of( ElementType.BYTE.toString(), ElementType.BYTE, ElementType.FLOAT.toString(), @@ -2502,7 +2502,7 @@ public String fieldName() { /** * @FunctionalInterface for a function that takes a int and boolean */ - interface IntBooleanConsumer { + public interface IntBooleanConsumer { void accept(int value, boolean isComplete); } } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 3dc25b058b1d6..09be98630d5c4 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -67,7 +67,6 @@ import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction; import org.elasticsearch.index.seqno.RetentionLeaseSyncAction; @@ -211,9 +210,6 @@ public static Map getMappers(List mappe mappers.put(DenseVectorFieldMapper.CONTENT_TYPE, DenseVectorFieldMapper.PARSER); mappers.put(SparseVectorFieldMapper.CONTENT_TYPE, SparseVectorFieldMapper.PARSER); - if (RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()) { - mappers.put(RankVectorsFieldMapper.CONTENT_TYPE, RankVectorsFieldMapper.PARSER); - } for (MapperPlugin mapperPlugin : mapperPlugins) { for (Map.Entry entry : mapperPlugin.getMappers().entrySet()) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java index 06f8f8f3c1be6..d9e2eaa8f92b6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java @@ -10,7 +10,6 @@ package org.elasticsearch.rest.action.search; import org.elasticsearch.Build; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; import java.util.HashSet; import java.util.Set; @@ -34,14 +33,8 @@ private SearchCapabilities() {} private static final String TRANSFORM_RANK_RRF_TO_RETRIEVER = "transform_rank_rrf_to_retriever"; /** Support kql query. */ private static final String KQL_QUERY_SUPPORTED = "kql_query"; - /** Support rank-vectors field mapper. */ - private static final String RANK_VECTORS_FIELD_MAPPER = "rank_vectors_field_mapper"; /** Support propagating nested retrievers' inner_hits to top-level compound retrievers . */ private static final String NESTED_RETRIEVER_INNER_HITS_SUPPORT = "nested_retriever_inner_hits_support"; - /** Support rank-vectors script field access. */ - private static final String RANK_VECTORS_SCRIPT_ACCESS = "rank_vectors_script_access"; - /** Initial support for rank-vectors maxSim functions access. */ - private static final String RANK_VECTORS_SCRIPT_MAX_SIM = "rank_vectors_script_max_sim_with_bugfix"; /** Fixed the math in {@code moving_fn}'s {@code linearWeightedAvg}. */ private static final String MOVING_FN_RIGHT_MATH = "moving_fn_right_math"; @@ -62,11 +55,6 @@ private SearchCapabilities() {} capabilities.add(OPTIMIZED_SCALAR_QUANTIZATION_BBQ); capabilities.add(KNN_QUANTIZED_VECTOR_RESCORE); capabilities.add(MOVING_FN_RIGHT_MATH); - if (RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()) { - capabilities.add(RANK_VECTORS_FIELD_MAPPER); - capabilities.add(RANK_VECTORS_SCRIPT_ACCESS); - capabilities.add(RANK_VECTORS_SCRIPT_MAX_SIM); - } if (Build.current().isSnapshot()) { capabilities.add(KQL_QUERY_SUPPORTED); } diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index f057d35d6e7a9..9e2dee4d94212 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -112,7 +112,6 @@ import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.IndexShard; @@ -206,7 +205,6 @@ public abstract class AggregatorTestCase extends ESTestCase { private static final List TYPE_TEST_BLACKLIST = List.of( ObjectMapper.CONTENT_TYPE, // Cannot aggregate objects DenseVectorFieldMapper.CONTENT_TYPE, // Cannot aggregate dense vectors - RankVectorsFieldMapper.CONTENT_TYPE, // Cannot aggregate dense vectors SparseVectorFieldMapper.CONTENT_TYPE, // Sparse vectors are no longer supported NestedObjectMapper.CONTENT_TYPE, // TODO support for nested diff --git a/x-pack/plugin/rank-vectors/build.gradle b/x-pack/plugin/rank-vectors/build.gradle new file mode 100644 index 0000000000000..327b380ddf867 --- /dev/null +++ b/x-pack/plugin/rank-vectors/build.gradle @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +apply plugin: 'elasticsearch.internal-es-plugin' +apply plugin: 'elasticsearch.internal-cluster-test' + +esplugin { + name 'rank-vectors' + description 'Rank vectors in search.' + classname 'org.elasticsearch.xpack.rank.vectors.RankVectorsPlugin' + extendedPlugins = ['x-pack-core', 'lang-painless'] +} + +dependencies { + compileOnly project(path: xpackModule('core')) + compileOnly(project(':modules:lang-painless:spi')) + + testImplementation(testArtifact(project(xpackModule('core')))) + testImplementation(testArtifact(project(':server'))) + + clusterModules project(':modules:lang-painless') +} diff --git a/x-pack/plugin/rank-vectors/src/main/java/module-info.java b/x-pack/plugin/rank-vectors/src/main/java/module-info.java new file mode 100644 index 0000000000000..85ca095d182c1 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/java/module-info.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module org.elasticsearch.rank.vectors { + requires org.elasticsearch.xcore; + requires org.elasticsearch.painless.spi; + requires org.elasticsearch.server; + requires org.elasticsearch.simdvec; + requires org.apache.lucene.core; + requires org.elasticsearch.xcontent; + + exports org.elasticsearch.xpack.rank.vectors; + exports org.elasticsearch.xpack.rank.vectors.mapper; + exports org.elasticsearch.xpack.rank.vectors.script; + + provides org.elasticsearch.painless.spi.PainlessExtension with org.elasticsearch.xpack.rank.vectors.script.RankVectorsPainlessExtension; + provides org.elasticsearch.features.FeatureSpecification with RankVectorsFeatures; + +} diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsFeatures.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsFeatures.java new file mode 100644 index 0000000000000..44b1b7a068860 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsFeatures.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rank.vectors; + +import org.elasticsearch.features.FeatureSpecification; +import org.elasticsearch.features.NodeFeature; + +import java.util.Set; + +public class RankVectorsFeatures implements FeatureSpecification { + public static final NodeFeature RANK_VECTORS_FEATURE = new NodeFeature("rank_vectors"); + + @Override + public Set getTestFeatures() { + return Set.of(RANK_VECTORS_FEATURE); + } + +} diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java new file mode 100644 index 0000000000000..8445b8a9268cc --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rank.vectors; + +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.license.License; +import org.elasticsearch.license.LicensedFeature; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsFieldMapper; + +import java.util.Map; + +public class RankVectorsPlugin extends Plugin implements MapperPlugin { + public static final LicensedFeature.Momentary RANK_VECTORS_FEATURE = LicensedFeature.momentary( + null, + "rank-vectors", + License.OperationMode.ENTERPRISE + ); + + + + @Override + public Map getMappers() { + return Map.of(RankVectorsFieldMapper.CONTENT_TYPE, RankVectorsFieldMapper.PARSER); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsDVLeafFieldData.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java similarity index 84% rename from server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsDVLeafFieldData.java rename to x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java index 0125d0249ec2b..52cd414bf7072 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsDVLeafFieldData.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java @@ -1,19 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.mapper.vectors; +package org.elasticsearch.xpack.rank.vectors.mapper; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReader; import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; import org.elasticsearch.script.field.vectors.BitRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapper.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java similarity index 92% rename from server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapper.java rename to x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java index d57dbf79b450c..07b398eb65792 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapper.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.mapper.vectors; +package org.elasticsearch.xpack.rank.vectors.mapper; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.index.BinaryDocValues; @@ -15,7 +13,6 @@ import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -31,12 +28,15 @@ import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.vectors.VectorData; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.core.XPackPlugin; import java.io.IOException; import java.nio.ByteBuffer; @@ -50,11 +50,11 @@ import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT_BIT; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.namesToElementType; +import static org.elasticsearch.xpack.rank.vectors.RankVectorsPlugin.RANK_VECTORS_FEATURE; public class RankVectorsFieldMapper extends FieldMapper { public static final String VECTOR_MAGNITUDES_SUFFIX = "._magnitude"; - public static final FeatureFlag FEATURE_FLAG = new FeatureFlag("rank_vectors"); public static final String CONTENT_TYPE = "rank_vectors"; private static RankVectorsFieldMapper toType(FieldMapper in) { @@ -122,12 +122,12 @@ protected Parameter[] getParameters() { return new Parameter[] { elementType, dims, meta }; } - public RankVectorsFieldMapper.Builder dimensions(int dimensions) { + public Builder dimensions(int dimensions) { this.dims.setValue(dimensions); return this; } - public RankVectorsFieldMapper.Builder elementType(DenseVectorFieldMapper.ElementType elementType) { + public Builder elementType(DenseVectorFieldMapper.ElementType elementType) { this.elementType.setValue(elementType); return this; } @@ -153,7 +153,12 @@ public RankVectorsFieldMapper build(MapperBuilderContext context) { } public static final TypeParser PARSER = new TypeParser( - (n, c) -> new RankVectorsFieldMapper.Builder(n, c.indexVersionCreated()), + (n, c) -> { + if (RANK_VECTORS_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) { + throw LicenseUtils.newComplianceException("Rank Vectors"); + } + return new Builder(n, c.indexVersionCreated()); + }, notInMultiFields(CONTENT_TYPE) ); @@ -207,7 +212,10 @@ public boolean isAggregatable() { @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { - return new RankVectorsIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, indexCreatedVersion, dims, elementType); + if (RANK_VECTORS_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) { + throw LicenseUtils.newComplianceException("Rank Vectors"); + } + return new RankVectorsIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, dims, elementType); } @Override @@ -366,12 +374,12 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new RankVectorsFieldMapper.Builder(leafName(), indexCreatedVersion).init(this); + return new Builder(leafName(), indexCreatedVersion).init(this); } @Override protected SyntheticSourceSupport syntheticSourceSupport() { - return new SyntheticSourceSupport.Native(new RankVectorsFieldMapper.DocValuesSyntheticFieldLoader()); + return new SyntheticSourceSupport.Native(new DocValuesSyntheticFieldLoader()); } private class DocValuesSyntheticFieldLoader extends SourceLoader.DocValuesBasedSyntheticFieldLoader { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsIndexFieldData.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsIndexFieldData.java similarity index 81% rename from server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsIndexFieldData.java rename to x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsIndexFieldData.java index 7f54d2b9a8ad8..7d74888ec0ef5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsIndexFieldData.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsIndexFieldData.java @@ -1,20 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.mapper.vectors; +package org.elasticsearch.xpack.rank.vectors.mapper; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.SortField; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; @@ -26,19 +24,16 @@ public class RankVectorsIndexFieldData implements IndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { - return new RankVectorsIndexFieldData(name, dims, valuesSourceType, indexVersion, elementType); + return new RankVectorsIndexFieldData(name, dims, valuesSourceType, elementType); } } } diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java new file mode 100644 index 0000000000000..1ff26c1c94368 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rank.vectors.script; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScoreScript; +import org.elasticsearch.script.ScriptContext; + +import java.util.List; +import java.util.Map; + +public class RankVectorsPainlessExtension implements PainlessExtension { + private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles(RankVectorsPainlessExtension.class, "whitelist.txt"); + + @Override + public Map, List> getContextWhitelists() { + return Map.of(ScoreScript.CONTEXT, List.of(WHITELIST)); + } +} diff --git a/server/src/main/java/org/elasticsearch/script/RankVectorsScoreScriptUtils.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java similarity index 96% rename from server/src/main/java/org/elasticsearch/script/RankVectorsScoreScriptUtils.java rename to x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java index 2d11641cb5aa7..abb5d4493f29b 100644 --- a/server/src/main/java/org/elasticsearch/script/RankVectorsScoreScriptUtils.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java @@ -7,17 +7,22 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.script; +package org.elasticsearch.xpack.rank.vectors.script; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.field.vectors.DenseVector; import org.elasticsearch.script.field.vectors.RankVectorsDocValuesField; +import org.elasticsearch.xpack.core.XPackPlugin; import java.io.IOException; import java.util.HexFormat; import java.util.List; +import static org.elasticsearch.xpack.rank.vectors.RankVectorsPlugin.RANK_VECTORS_FEATURE; + public class RankVectorsScoreScriptUtils { public static class RankVectorsFunction { @@ -179,6 +184,9 @@ public static final class MaxSimInvHamming { private final MaxSimInvHammingDistanceInterface function; public MaxSimInvHamming(ScoreScript scoreScript, Object queryVector, String fieldName) { + if (RANK_VECTORS_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) { + throw LicenseUtils.newComplianceException("Rank Vectors"); + } RankVectorsDocValuesField field = (RankVectorsDocValuesField) scoreScript.field(fieldName); if (field.getElementType() == DenseVectorFieldMapper.ElementType.FLOAT) { throw new IllegalArgumentException("hamming distance is only supported for byte or bit vectors"); @@ -334,6 +342,9 @@ public static final class MaxSimDotProduct { @SuppressWarnings("unchecked") public MaxSimDotProduct(ScoreScript scoreScript, Object queryVector, String fieldName) { + if (RANK_VECTORS_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) { + throw LicenseUtils.newComplianceException("Rank Vectors"); + } RankVectorsDocValuesField field = (RankVectorsDocValuesField) scoreScript.field(fieldName); function = switch (field.getElementType()) { case BIT -> { diff --git a/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification new file mode 100644 index 0000000000000..388dbe7dead96 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -0,0 +1,8 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +org.elasticsearch.xpack.rank.vectors.RankVectorsFeatures diff --git a/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..06c77434c77a6 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.xpack.rank.vectors.script.RankVectorsPainlessExtension diff --git a/x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/whitelist.txt b/x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/whitelist.txt new file mode 100644 index 0000000000000..ce3d04f73dea8 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/whitelist.txt @@ -0,0 +1,13 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +# This file contains a rank vectors whitelist for functions to be used in Score context + +static_import { + double maxSimDotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.xpack.rank.vectors.script.RankVectorsScoreScriptUtils$MaxSimDotProduct + double maxSimInvHamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.xpack.rank.vectors.script.RankVectorsScoreScriptUtils$MaxSimInvHamming +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapperTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java similarity index 97% rename from server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapperTests.java rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java index e81c28cbcc444..7d9ad83eca925 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapperTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.mapper.vectors; +package org.elasticsearch.xpack.rank.vectors.mapper; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.index.IndexableField; @@ -34,7 +32,6 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.junit.AssumptionViolatedException; -import org.junit.BeforeClass; import java.io.IOException; import java.nio.ByteBuffer; @@ -53,11 +50,6 @@ public class RankVectorsFieldMapperTests extends MapperTestCase { - @BeforeClass - public static void setup() { - assumeTrue("Requires rank vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); - } - private final ElementType elementType; private final int dims; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldTypeTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java similarity index 80% rename from server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldTypeTests.java rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java index b4cbbc4730d7c..d33317fbd2ada 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldTypeTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java @@ -1,20 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.mapper.vectors; +package org.elasticsearch.xpack.rank.vectors.mapper; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper.RankVectorsFieldType; -import org.junit.BeforeClass; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsFieldMapper.RankVectorsFieldType; import java.io.IOException; import java.util.Collections; @@ -25,11 +23,6 @@ public class RankVectorsFieldTypeTests extends FieldTypeTestCase { - @BeforeClass - public static void setup() { - assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); - } - private RankVectorsFieldType createFloatFieldType() { return new RankVectorsFieldType( "f", @@ -40,7 +33,7 @@ private RankVectorsFieldType createFloatFieldType() { ); } - private RankVectorsFieldType createByteFieldType() { + private RankVectorsFieldMapper.RankVectorsFieldType createByteFieldType() { return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.BYTE, 5, IndexVersion.current(), Collections.emptyMap()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValuesTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsScriptDocValuesTests.java similarity index 95% rename from server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValuesTests.java rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsScriptDocValuesTests.java index c38ed0f60f0ae..f0b00849557bd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValuesTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsScriptDocValuesTests.java @@ -1,24 +1,22 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.mapper.vectors; +package org.elasticsearch.xpack.rank.vectors.mapper; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; +import org.elasticsearch.index.mapper.vectors.RankVectorsScriptDocValues; import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.RankVectors; import org.elasticsearch.script.field.vectors.RankVectorsDocValuesField; import org.elasticsearch.test.ESTestCase; -import org.junit.BeforeClass; import java.io.IOException; import java.nio.ByteBuffer; @@ -29,11 +27,6 @@ public class RankVectorsScriptDocValuesTests extends ESTestCase { - @BeforeClass - public static void setup() { - assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); - } - public void testFloatGetVectorValueAndGetMagnitude() throws IOException { int dims = 3; float[][][] vectors = { { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } }, { { 1, 0, 2 } } }; diff --git a/server/src/test/java/org/elasticsearch/script/RankVectorsScoreScriptUtilsTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtilsTests.java similarity index 93% rename from server/src/test/java/org/elasticsearch/script/RankVectorsScoreScriptUtilsTests.java rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtilsTests.java index 917cc2069a293..da0340a22c074 100644 --- a/server/src/test/java/org/elasticsearch/script/RankVectorsScoreScriptUtilsTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtilsTests.java @@ -1,26 +1,23 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.script; +package org.elasticsearch.xpack.rank.vectors.script; import org.apache.lucene.util.VectorUtil; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; -import org.elasticsearch.index.mapper.vectors.RankVectorsScriptDocValuesTests; -import org.elasticsearch.script.RankVectorsScoreScriptUtils.MaxSimDotProduct; -import org.elasticsearch.script.RankVectorsScoreScriptUtils.MaxSimInvHamming; +import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.field.vectors.BitRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.RankVectorsDocValuesField; import org.elasticsearch.test.ESTestCase; -import org.junit.BeforeClass; +import org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsScriptDocValuesTests; +import org.elasticsearch.xpack.rank.vectors.script.RankVectorsScoreScriptUtils.MaxSimDotProduct; +import org.elasticsearch.xpack.rank.vectors.script.RankVectorsScoreScriptUtils.MaxSimInvHamming; import java.io.IOException; import java.util.Arrays; @@ -33,11 +30,6 @@ public class RankVectorsScoreScriptUtilsTests extends ESTestCase { - @BeforeClass - public static void setup() { - assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); - } - public void testFloatMultiVectorClassBindings() throws IOException { String fieldName = "vector"; int dims = 5; @@ -88,7 +80,7 @@ public void testFloatMultiVectorClassBindings() throws IOException { // Check each function rejects query vectors with the wrong dimension IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> new RankVectorsScoreScriptUtils.MaxSimDotProduct(scoreScript, invalidQueryVector, fieldName) + () -> new MaxSimDotProduct(scoreScript, invalidQueryVector, fieldName) ); assertThat( e.getMessage(), @@ -336,7 +328,4 @@ public void testByteBoundaries() throws IOException { } } - public void testDimMismatch() throws IOException { - - } } diff --git a/server/src/test/java/org/elasticsearch/script/field/vectors/RankVectorsTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsTests.java similarity index 80% rename from server/src/test/java/org/elasticsearch/script/field/vectors/RankVectorsTests.java rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsTests.java index ca7608b10aed9..570e2a59926aa 100644 --- a/server/src/test/java/org/elasticsearch/script/field/vectors/RankVectorsTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsTests.java @@ -1,19 +1,19 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.script.field.vectors; +package org.elasticsearch.xpack.rank.vectors.script; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.VectorUtil; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; +import org.elasticsearch.script.field.vectors.ByteRankVectors; +import org.elasticsearch.script.field.vectors.FloatRankVectors; +import org.elasticsearch.script.field.vectors.RankVectors; +import org.elasticsearch.script.field.vectors.VectorIterator; import org.elasticsearch.test.ESTestCase; -import org.junit.BeforeClass; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -21,11 +21,6 @@ public class RankVectorsTests extends ESTestCase { - @BeforeClass - public static void setup() { - assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); - } - public void testByteUnsupported() { int count = randomIntBetween(1, 16); int dims = randomIntBetween(1, 16); diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_rank_vectors.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml similarity index 92% rename from rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_rank_vectors.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml index ecf34f46c3383..791712ee925a5 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_rank_vectors.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml @@ -1,11 +1,7 @@ setup: - requires: - capabilities: - - method: POST - path: /_search - capabilities: [ rank_vectors_field_mapper ] - test_runner_features: capabilities - reason: "Support for rank vectors field mapper capability required" + cluster_features: [ "rank_vectors" ] + reason: "requires rank_vectors feature" --- "Test create multi-vector field": - do: diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_rank_vectors_dv_fields_api.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml similarity index 94% rename from modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_rank_vectors_dv_fields_api.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml index f37e554fca7bf..d5fe9b3ac1e9c 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_rank_vectors_dv_fields_api.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml @@ -1,11 +1,7 @@ setup: - requires: - capabilities: - - method: POST - path: /_search - capabilities: [ rank_vectors_script_access ] - test_runner_features: capabilities - reason: "Support for rank vector field script access capability required" + cluster_features: [ "rank_vectors" ] + reason: "requires rank_vectors feature" - skip: features: headers diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_rank_vectors_max_sim.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_max_sim.yml similarity index 95% rename from modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_rank_vectors_max_sim.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_max_sim.yml index 7c46fbc9a26a5..acaf1b99b626e 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_rank_vectors_max_sim.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_max_sim.yml @@ -1,11 +1,7 @@ setup: - requires: - capabilities: - - method: POST - path: /_search - capabilities: [ rank_vectors_script_max_sim_with_bugfix ] - test_runner_features: capabilities - reason: "Support for rank vectors max-sim functions capability required" + cluster_features: [ "rank_vectors" ] + reason: "requires rank_vectors feature" - skip: features: headers From ef1506872691fb6e3f7299058f1b77c6a3d68b31 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 16 Dec 2024 17:08:34 -0500 Subject: [PATCH 02/14] cleanups and docs --- docs/reference/mapping/types.asciidoc | 2 + .../mapping/types/dense-vector.asciidoc | 1 - .../mapping/types/rank-vectors.asciidoc | 183 ++++++++++++++++++ .../vectors/DenseVectorFieldMapper.java | 24 ++- x-pack/plugin/rank-vectors/build.gradle | 2 - .../src/main/java/module-info.java | 3 +- .../xpack/rank/vectors/RankVectorsPlugin.java | 2 - .../mapper/RankVectorsFieldMapper.java | 48 ++--- .../mapper/RankVectorsIndexFieldData.java | 7 +- .../script/RankVectorsPainlessExtension.java | 5 +- ...hitelist.txt => rank_vector_whitelist.txt} | 0 .../mapper/RankVectorsFieldTypeTests.java | 11 +- 12 files changed, 223 insertions(+), 65 deletions(-) create mode 100644 docs/reference/mapping/types/rank-vectors.asciidoc rename x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/{whitelist.txt => rank_vector_whitelist.txt} (100%) diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index babe4f508b5f0..e5155b7d4ce5b 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -180,6 +180,8 @@ include::types/rank-feature.asciidoc[] include::types/rank-features.asciidoc[] +include::types/rank-vectors.asciidoc[] + include::types/search-as-you-type.asciidoc[] include::types/semantic-text.asciidoc[] diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index 199a59a5b143c..c16b979043a57 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -1,4 +1,3 @@ -[role="xpack"] [[dense-vector]] === Dense vector field type ++++ diff --git a/docs/reference/mapping/types/rank-vectors.asciidoc b/docs/reference/mapping/types/rank-vectors.asciidoc new file mode 100644 index 0000000000000..f0964d48cf124 --- /dev/null +++ b/docs/reference/mapping/types/rank-vectors.asciidoc @@ -0,0 +1,183 @@ +[role="xpack"] +[[rank-vectors]] +=== Rank Vectors +++++ + Rank Vectors +++++ +experimental::[] + +The `rank_vectors` field type enables late-interaction dense vector scoring in Elasticsearch. The number of vectors +per field can vary, but they must all share the same number of dimensions and element type. + +The purpose of vectors stored in this field is second order ranking documents with max-sim similarity. + +Here is a simple example of using this field with `float` elements. + +[source,console] +-------------------------------------------------- +PUT my-rank-vectors-float +{ + "mappings": { + "properties": { + "my_vector": { + "type": "rank_vector" + } + } + } +} + +PUT my-rank-vectors-float/_doc/1 +{ + "my_vector" : [[0.5, 10, 6], [-0.5, 10, 10]] +} + +-------------------------------------------------- + +In addition to the `float` element type, `byte` and `bit` element types are also supported. + +Here is an example of using this field with `byte` elements. + +[source,console] +-------------------------------------------------- +PUT my-rank-vectors-byte +{ + "mappings": { + "properties": { + "my_vector": { + "type": "rank_vector", + "element_type": "byte" + } + } + } +} + +PUT my-rank-vectors-byte/_doc/1 +{ + "my_vector" : [[1, 2, 3], [4, 5, 6]] +} +-------------------------------------------------- + +Here is an example of using this field with `bit` elements. + +[source,console] +-------------------------------------------------- +PUT my-rank-vectors-bit +{ + "mappings": { + "properties": { + "my_vector": { + "type": "rank_vector", + "element_type": "bit" + } + } + } +} + +POST /my-bit-vectors/_bulk?refresh +{"index": {"_id" : "1"}} +{"my_vector": [127, -127, 0, 1, 42]} +{"index": {"_id" : "2"}} +{"my_vector": "8100012a7f"} +-------------------------------------------------- + + +[role="child_attributes"] +[[rank-vectors-params]] +==== Parameters for rank vectors fields + +The `rank_vectors` field type supports the following parameters: + +[[rank-vectors-element-type]] +`element_type`:: +(Optional, string) +The data type used to encode vectors. The supported data types are +`float` (default), `byte`, and bit. + +.Valid values for `element_type` +[%collapsible%open] +==== +`float`::: +indexes a 4-byte floating-point +value per dimension. This is the default value. + +`byte`::: +indexes a 1-byte integer value per dimension. + +`bit`::: +indexes a single bit per dimension. Useful for very high-dimensional vectors or models that specifically support bit vectors. +NOTE: when using `bit`, the number of dimensions must be a multiple of 8 and must represent the number of bits. + +==== + +`dims`:: +(Optional, integer) +Number of vector dimensions. Can't exceed `4096`. If `dims` is not specified, +it will be set to the length of the first vector added to the field. + +[[rank-vectors-synthetic-source]] +==== Synthetic `_source` + +IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices +(indices that have `index.mode` set to `time_series`). For other indices +synthetic `_source` is in technical preview. Features in technical preview may +be changed or removed in a future release. Elastic will work to fix +any issues, but features in technical preview are not subject to the support SLA +of official GA features. + +`rank_vectors` fields support <> . + +[[rank-vectors-scoring]] +==== Scoring with rank vectors + +Rank vectors can be accessed and used in <>. + +For example, the following query scores documents based on the maxSim similarity between the query vector and the vectors stored in the `my_vector` field: + +[source,console] +-------------------------------------------------- +GET my-index/_search +{ + "query": { + "script_score": { + "query": { + "match_all": {} + }, + "script": { + "source": "maxSimDotProduct(params.query_vector, 'my_vector')", + "params": { + "query_vector": [[0.5, 10, 6], [-0.5, 10, 10]] + } + } + } + } +} +-------------------------------------------------- + +Additionally, asymmetric similarity functions can be used to score against `bit` vectors. For example, the following query scores documents based on the maxSimDotProduct similarity between a floating point query vector and bit vectors stored in the `my_vector` field: + +[source,console] +-------------------------------------------------- +GET my-index/_search +{ + "query": { + "script_score": { + "query": { + "match_all": {} + }, + "script": { + "source": "maxSimDotProduct(params.query_vector, 'my_vector')", + "params": { + "query_vector": [ + [0.35, 0.77, 0.95, 0.15, 0.11, 0.08, 0.58, 0.06, 0.44, 0.52, 0.21, + 0.62, 0.65, 0.16, 0.64, 0.39, 0.93, 0.06, 0.93, 0.31, 0.92, 0.0, + 0.66, 0.86, 0.92, 0.03, 0.81, 0.31, 0.2 , 0.92, 0.95, 0.64, 0.19, + 0.26, 0.77, 0.64, 0.78, 0.32, 0.97, 0.84] + ] <1> + } + } + } + } +} +-------------------------------------------------- +<1> Note that the query vector has 40 elements, matching the number of bits in the bit vectors. + diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index a368329eaea50..0be20babe1f25 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -486,8 +486,12 @@ private VectorData parseHexEncodedVector( } @Override - public VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) - throws IOException { + public VectorData parseKnnVector( + DocumentParserContext context, + int dims, + IntBooleanConsumer dimChecker, + VectorSimilarity similarity + ) throws IOException { XContentParser.Token token = context.parser().currentToken(); return switch (token) { case START_ARRAY -> parseVectorArray(context, dims, dimChecker, similarity); @@ -691,8 +695,12 @@ && isNotUnitVector(squaredMagnitude)) { } @Override - public VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) - throws IOException { + public VectorData parseKnnVector( + DocumentParserContext context, + int dims, + IntBooleanConsumer dimChecker, + VectorSimilarity similarity + ) throws IOException { int index = 0; float squaredMagnitude = 0; float[] vector = new float[dims]; @@ -889,8 +897,12 @@ private VectorData parseHexEncodedVector(DocumentParserContext context, IntBoole } @Override - public VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) - throws IOException { + public VectorData parseKnnVector( + DocumentParserContext context, + int dims, + IntBooleanConsumer dimChecker, + VectorSimilarity similarity + ) throws IOException { XContentParser.Token token = context.parser().currentToken(); return switch (token) { case START_ARRAY -> parseVectorArray(context, dims, dimChecker, similarity); diff --git a/x-pack/plugin/rank-vectors/build.gradle b/x-pack/plugin/rank-vectors/build.gradle index 327b380ddf867..cd550d96a5503 100644 --- a/x-pack/plugin/rank-vectors/build.gradle +++ b/x-pack/plugin/rank-vectors/build.gradle @@ -21,6 +21,4 @@ dependencies { testImplementation(testArtifact(project(xpackModule('core')))) testImplementation(testArtifact(project(':server'))) - - clusterModules project(':modules:lang-painless') } diff --git a/x-pack/plugin/rank-vectors/src/main/java/module-info.java b/x-pack/plugin/rank-vectors/src/main/java/module-info.java index 85ca095d182c1..0b00aeb14c0d9 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/module-info.java +++ b/x-pack/plugin/rank-vectors/src/main/java/module-info.java @@ -9,7 +9,6 @@ requires org.elasticsearch.xcore; requires org.elasticsearch.painless.spi; requires org.elasticsearch.server; - requires org.elasticsearch.simdvec; requires org.apache.lucene.core; requires org.elasticsearch.xcontent; @@ -18,6 +17,6 @@ exports org.elasticsearch.xpack.rank.vectors.script; provides org.elasticsearch.painless.spi.PainlessExtension with org.elasticsearch.xpack.rank.vectors.script.RankVectorsPainlessExtension; - provides org.elasticsearch.features.FeatureSpecification with RankVectorsFeatures; + provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.rank.vectors.RankVectorsFeatures; } diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java index 8445b8a9268cc..0158e362228df 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java @@ -23,8 +23,6 @@ public class RankVectorsPlugin extends Plugin implements MapperPlugin { License.OperationMode.ENTERPRISE ); - - @Override public Map getMappers() { return Map.of(RankVectorsFieldMapper.CONTENT_TYPE, RankVectorsFieldMapper.PARSER); diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java index 07b398eb65792..4ae926a968c19 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java @@ -122,62 +122,39 @@ protected Parameter[] getParameters() { return new Parameter[] { elementType, dims, meta }; } - public Builder dimensions(int dimensions) { - this.dims.setValue(dimensions); - return this; - } - - public Builder elementType(DenseVectorFieldMapper.ElementType elementType) { - this.elementType.setValue(elementType); - return this; - } - @Override public RankVectorsFieldMapper build(MapperBuilderContext context) { + // Validate on Mapping creation + if (RANK_VECTORS_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) { + throw LicenseUtils.newComplianceException("Rank Vectors"); + } // Validate again here because the dimensions or element type could have been set programmatically, // which affects index option validity validate(); return new RankVectorsFieldMapper( leafName(), - new RankVectorsFieldType( - context.buildFullName(leafName()), - elementType.getValue(), - dims.getValue(), - indexCreatedVersion, - meta.getValue() - ), + new RankVectorsFieldType(context.buildFullName(leafName()), elementType.getValue(), dims.getValue(), meta.getValue()), builderParams(this, context), indexCreatedVersion ); } } - public static final TypeParser PARSER = new TypeParser( - (n, c) -> { - if (RANK_VECTORS_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) { - throw LicenseUtils.newComplianceException("Rank Vectors"); - } - return new Builder(n, c.indexVersionCreated()); - }, - notInMultiFields(CONTENT_TYPE) - ); + public static final TypeParser PARSER = new TypeParser((n, c) -> { + if (RANK_VECTORS_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) { + throw LicenseUtils.newComplianceException("Rank Vectors"); + } + return new Builder(n, c.indexVersionCreated()); + }, notInMultiFields(CONTENT_TYPE)); public static final class RankVectorsFieldType extends SimpleMappedFieldType { private final DenseVectorFieldMapper.ElementType elementType; private final Integer dims; - private final IndexVersion indexCreatedVersion; - public RankVectorsFieldType( - String name, - DenseVectorFieldMapper.ElementType elementType, - Integer dims, - IndexVersion indexCreatedVersion, - Map meta - ) { + public RankVectorsFieldType(String name, DenseVectorFieldMapper.ElementType elementType, Integer dims, Map meta) { super(name, false, false, true, TextSearchInfo.NONE, meta); this.elementType = elementType; this.dims = dims; - this.indexCreatedVersion = indexCreatedVersion; } @Override @@ -289,7 +266,6 @@ public void parse(DocumentParserContext context) throws IOException { fieldType().name(), fieldType().elementType, currentDims, - indexCreatedVersion, fieldType().meta() ); Mapper update = new RankVectorsFieldMapper(leafName(), updatedFieldType, builderParams, indexCreatedVersion); diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsIndexFieldData.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsIndexFieldData.java index 7d74888ec0ef5..7ec426b904ce0 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsIndexFieldData.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsIndexFieldData.java @@ -86,12 +86,7 @@ public static class Builder implements IndexFieldData.Builder { private final int dims; private final DenseVectorFieldMapper.ElementType elementType; - public Builder( - String name, - ValuesSourceType valuesSourceType, - int dims, - DenseVectorFieldMapper.ElementType elementType - ) { + public Builder(String name, ValuesSourceType valuesSourceType, int dims, DenseVectorFieldMapper.ElementType elementType) { this.name = name; this.valuesSourceType = valuesSourceType; this.dims = dims; diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java index 1ff26c1c94368..5893452f5fdf8 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java @@ -17,7 +17,10 @@ import java.util.Map; public class RankVectorsPainlessExtension implements PainlessExtension { - private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles(RankVectorsPainlessExtension.class, "whitelist.txt"); + private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles( + RankVectorsPainlessExtension.class, + "rank_vector_whitelist.txt" + ); @Override public Map, List> getContextWhitelists() { diff --git a/x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/whitelist.txt b/x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/rank_vector_whitelist.txt similarity index 100% rename from x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/whitelist.txt rename to x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/rank_vector_whitelist.txt diff --git a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java index d33317fbd2ada..30de24939bb24 100644 --- a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.rank.vectors.mapper; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; @@ -24,17 +23,11 @@ public class RankVectorsFieldTypeTests extends FieldTypeTestCase { private RankVectorsFieldType createFloatFieldType() { - return new RankVectorsFieldType( - "f", - DenseVectorFieldMapper.ElementType.FLOAT, - BBQ_MIN_DIMS, - IndexVersion.current(), - Collections.emptyMap() - ); + return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.FLOAT, BBQ_MIN_DIMS, Collections.emptyMap()); } private RankVectorsFieldMapper.RankVectorsFieldType createByteFieldType() { - return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.BYTE, 5, IndexVersion.current(), Collections.emptyMap()); + return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.BYTE, 5, Collections.emptyMap()); } public void testHasDocValues() { From 9a65da8a767d82432b4bbaee5ad8c217db95e044 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Mon, 16 Dec 2024 17:17:08 -0500 Subject: [PATCH 03/14] Update docs/changelog/118804.yaml --- docs/changelog/118804.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 docs/changelog/118804.yaml diff --git a/docs/changelog/118804.yaml b/docs/changelog/118804.yaml new file mode 100644 index 0000000000000..d0edbbcb69ecc --- /dev/null +++ b/docs/changelog/118804.yaml @@ -0,0 +1,28 @@ +pr: 118804 +summary: Add new experimental `rank_vectors` mapping for late-interaction second order + ranking +area: Vector Search +type: feature +issues: [] +highlight: + title: Add new experimental `rank_vectors` mapping for late-interaction second order + ranking + body: "Late-interaction models are powerful rerankers. While their size and\noverall\ + \ cost doesn't lend itself for HNSW indexing, utilizing them as\nsecond order\ + \ \"brute-force\" reranking can provide excellent boosts in\nrelevance. At generally\ + \ lower inference times than large cross-encoders.\n\n\nThis commit exposes a\ + \ new experimental `rank_vectors` field that allows\nfor maxSim operations. This\ + \ unlocks the initial, and most common use of\nlate-interaction dense-models.\ + \ \n\nFor example, this is how you would use it via the API:\n\n```\nPUT index\n\ + {\n \"mappings\": {\n \"properties\": {\n \"late_interaction_vectors\"\ + : {\n \"type\": \"rank_vectors\"\n }\n }\n }\n}\n```\n\nThen to\ + \ index:\n\n```\nPOST index/_doc\n{\n \"late_interaction_vectors\": [[0.1, ...],...]\n\ + }\n```\n\nFor querying, scoring can be exposed with scripting:\n\n```\nPOST index/_search\n\ + {\n \"query\": {\n \"script_score\": {\n \"query\": {\n \"match_all\"\ + : {}\n },\n \"script\": {\n \"source\": \"maxSimDotProduct(params.query_vector,\ + \ 'my_vector')\",\n \"params\": {\n \"query_vector\": [[0.42,\ + \ ...], ...]\n }\n }\n }\n }\n}\n```\n\nOf course, the initial\ + \ ranking should be done before re-scoring or\ncombining via the `rescore` parameter,\ + \ or simply passing whatever first\nphase retrieval you want as the inner query\ + \ in `script_score`." + notable: true From 023f11ff4d4040288536df939f260d2e024fb178 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:28:30 -0500 Subject: [PATCH 04/14] giving whitelist resource access to painless spi --- x-pack/plugin/rank-vectors/src/main/java/module-info.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugin/rank-vectors/src/main/java/module-info.java b/x-pack/plugin/rank-vectors/src/main/java/module-info.java index 0b00aeb14c0d9..1f14baf7525ab 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/module-info.java +++ b/x-pack/plugin/rank-vectors/src/main/java/module-info.java @@ -16,6 +16,9 @@ exports org.elasticsearch.xpack.rank.vectors.mapper; exports org.elasticsearch.xpack.rank.vectors.script; + // whitelist resource access + opens org.elasticsearch.xpack.rank.vectors.script to org.elasticsearch.painless.spi; + provides org.elasticsearch.painless.spi.PainlessExtension with org.elasticsearch.xpack.rank.vectors.script.RankVectorsPainlessExtension; provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.rank.vectors.RankVectorsFeatures; From 341662a92deb3130cdbed6b52d4d2af8d0925379 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 17 Dec 2024 16:35:54 +0000 Subject: [PATCH 05/14] [CI] Auto commit changes from spotless --- x-pack/plugin/rank-vectors/src/main/java/module-info.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/rank-vectors/src/main/java/module-info.java b/x-pack/plugin/rank-vectors/src/main/java/module-info.java index 1f14baf7525ab..4af3c994edd38 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/module-info.java +++ b/x-pack/plugin/rank-vectors/src/main/java/module-info.java @@ -17,7 +17,7 @@ exports org.elasticsearch.xpack.rank.vectors.script; // whitelist resource access - opens org.elasticsearch.xpack.rank.vectors.script to org.elasticsearch.painless.spi; + opens org.elasticsearch.xpack.rank.vectors.script to org.elasticsearch.painless.spi; provides org.elasticsearch.painless.spi.PainlessExtension with org.elasticsearch.xpack.rank.vectors.script.RankVectorsPainlessExtension; provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.rank.vectors.RankVectorsFeatures; From fedd90a2724c8b89d371f0fdee7d82572ce689b5 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:24:47 -0500 Subject: [PATCH 06/14] fixing tests and headers --- docs/reference/mapping/types/rank-vectors.asciidoc | 6 +++--- .../rank/vectors/script/RankVectorsScoreScriptUtils.java | 8 +++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/reference/mapping/types/rank-vectors.asciidoc b/docs/reference/mapping/types/rank-vectors.asciidoc index f0964d48cf124..14668c03c5bd6 100644 --- a/docs/reference/mapping/types/rank-vectors.asciidoc +++ b/docs/reference/mapping/types/rank-vectors.asciidoc @@ -20,7 +20,7 @@ PUT my-rank-vectors-float "mappings": { "properties": { "my_vector": { - "type": "rank_vector" + "type": "rank_vectors" } } } @@ -44,7 +44,7 @@ PUT my-rank-vectors-byte "mappings": { "properties": { "my_vector": { - "type": "rank_vector", + "type": "rank_vectors", "element_type": "byte" } } @@ -66,7 +66,7 @@ PUT my-rank-vectors-bit "mappings": { "properties": { "my_vector": { - "type": "rank_vector", + "type": "rank_vectors", "element_type": "bit" } } diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java index abb5d4493f29b..722344d7cfe0a 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java @@ -1,10 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.xpack.rank.vectors.script; From 01cb42598cdbcfde777fd4c72138f681d72826dc Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:36:01 -0500 Subject: [PATCH 07/14] fixing notes --- docs/changelog/118804.yaml | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/docs/changelog/118804.yaml b/docs/changelog/118804.yaml index d0edbbcb69ecc..1548367a5485f 100644 --- a/docs/changelog/118804.yaml +++ b/docs/changelog/118804.yaml @@ -7,22 +7,9 @@ issues: [] highlight: title: Add new experimental `rank_vectors` mapping for late-interaction second order ranking - body: "Late-interaction models are powerful rerankers. While their size and\noverall\ - \ cost doesn't lend itself for HNSW indexing, utilizing them as\nsecond order\ - \ \"brute-force\" reranking can provide excellent boosts in\nrelevance. At generally\ - \ lower inference times than large cross-encoders.\n\n\nThis commit exposes a\ - \ new experimental `rank_vectors` field that allows\nfor maxSim operations. This\ - \ unlocks the initial, and most common use of\nlate-interaction dense-models.\ - \ \n\nFor example, this is how you would use it via the API:\n\n```\nPUT index\n\ - {\n \"mappings\": {\n \"properties\": {\n \"late_interaction_vectors\"\ - : {\n \"type\": \"rank_vectors\"\n }\n }\n }\n}\n```\n\nThen to\ - \ index:\n\n```\nPOST index/_doc\n{\n \"late_interaction_vectors\": [[0.1, ...],...]\n\ - }\n```\n\nFor querying, scoring can be exposed with scripting:\n\n```\nPOST index/_search\n\ - {\n \"query\": {\n \"script_score\": {\n \"query\": {\n \"match_all\"\ - : {}\n },\n \"script\": {\n \"source\": \"maxSimDotProduct(params.query_vector,\ - \ 'my_vector')\",\n \"params\": {\n \"query_vector\": [[0.42,\ - \ ...], ...]\n }\n }\n }\n }\n}\n```\n\nOf course, the initial\ - \ ranking should be done before re-scoring or\ncombining via the `rescore` parameter,\ - \ or simply passing whatever first\nphase retrieval you want as the inner query\ - \ in `script_score`." + body: + Late-interaction models are powerful rerankers. While their size and overall + cost doesn't lend itself for HNSW indexing, utilizing them as second order reranking + can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring + over new and novel multi-vector late-interaction models like ColBERT or ColPali. notable: true From 0155f60ad069300c604b3f2a989d95365bcdb868 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:41:37 -0500 Subject: [PATCH 08/14] adding tests and doc values --- .../ByteRankVectorsDocValuesField.java | 4 +- .../FloatRankVectorsDocValuesField.java | 4 +- .../mapper/RankVectorsDVLeafFieldData.java | 98 +++++++++++++++++++ .../mapper/RankVectorsFieldMapper.java | 4 +- .../rank_vectors_dv_fields_api.yml | 36 +++++++ 5 files changed, 139 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java index db81bb6ebe1cb..1bff1b50fb5ac 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java @@ -111,13 +111,13 @@ public boolean isEmpty() { return value == null; } - static class ByteVectorIterator implements VectorIterator { + public static class ByteVectorIterator implements VectorIterator { private final byte[] buffer; private final BytesRef vectorValues; private final int size; private int idx = 0; - ByteVectorIterator(BytesRef vectorValues, byte[] buffer, int size) { + public ByteVectorIterator(BytesRef vectorValues, byte[] buffer, int size) { assert vectorValues.length == (buffer.length * size); this.vectorValues = vectorValues; this.size = size; diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java index 39bc1e621113b..d47795a3b2401 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java @@ -110,14 +110,14 @@ private void decodeVectorIfNecessary() { } } - static class FloatVectorIterator implements VectorIterator { + public static class FloatVectorIterator implements VectorIterator { private final float[] buffer; private final FloatBuffer vectorValues; private final BytesRef vectorValueBytesRef; private final int size; private int idx = 0; - FloatVectorIterator(BytesRef vectorValues, float[] buffer, int size) { + public FloatVectorIterator(BytesRef vectorValues, float[] buffer, int size) { assert vectorValues.length == (buffer.length * Float.BYTES * size); this.vectorValueBytesRef = vectorValues; this.vectorValues = ByteBuffer.wrap(vectorValues.bytes, vectorValues.offset, vectorValues.length) diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java index 52cd414bf7072..b858b935c1483 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java @@ -10,6 +10,8 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.fielddata.FormattedDocValues; import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; @@ -17,8 +19,13 @@ import org.elasticsearch.script.field.vectors.BitRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField; +import org.elasticsearch.script.field.vectors.VectorIterator; +import org.elasticsearch.search.DocValueFormat; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; final class RankVectorsDVLeafFieldData implements LeafFieldData { private final LeafReader reader; @@ -33,6 +40,97 @@ final class RankVectorsDVLeafFieldData implements LeafFieldData { this.dims = dims; } + @Override + public FormattedDocValues getFormattedValues(DocValueFormat format) { + int dims = elementType == DenseVectorFieldMapper.ElementType.BIT ? this.dims / Byte.SIZE : this.dims; + return switch (elementType) { + case BYTE, BIT -> new FormattedDocValues() { + private final byte[] vector = new byte[dims]; + private BytesRef ref = null; + private int numVecs = -1; + private final BinaryDocValues binary; + { + try { + binary = DocValues.getBinary(reader, field); + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } + + @Override + public boolean advanceExact(int docId) throws IOException { + if (binary == null || binary.advanceExact(docId) == false) { + return false; + } + ref = binary.binaryValue(); + assert ref.length % dims == 0; + numVecs = ref.length / dims; + return true; + } + + @Override + public int docValueCount() { + return 1; + } + + public Object nextValue() { + // Boxed to keep from `byte[]` being transformed into a string + List vectors = new ArrayList<>(numVecs); + VectorIterator iterator = new ByteRankVectorsDocValuesField.ByteVectorIterator(ref, vector, numVecs); + while (iterator.hasNext()) { + byte[] v = iterator.next(); + Byte[] vec = new Byte[dims]; + for (int i = 0; i < dims; i++) { + vec[i] = v[i]; + } + vectors.add(vec); + } + return vectors; + } + }; + case FLOAT -> new FormattedDocValues() { + private final float[] vector = new float[dims]; + private BytesRef ref = null; + private int numVecs = -1; + private final BinaryDocValues binary; + { + try { + binary = DocValues.getBinary(reader, field); + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } + + @Override + public boolean advanceExact(int docId) throws IOException { + if (binary == null || binary.advanceExact(docId) == false) { + return false; + } + ref = binary.binaryValue(); + assert ref.length % (Float.BYTES * dims) == 0; + numVecs = ref.length / (Float.BYTES * dims); + return true; + } + + @Override + public int docValueCount() { + return 1; + } + + @Override + public Object nextValue() { + List vectors = new ArrayList<>(numVecs); + VectorIterator iterator = new FloatRankVectorsDocValuesField.FloatVectorIterator(ref, vector, numVecs); + while (iterator.hasNext()) { + float[] v = iterator.next(); + vectors.add(Arrays.copyOf(v, v.length)); + } + return vectors; + } + }; + }; + } + @Override public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { try { diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java index 4ae926a968c19..4e4d4ebac8fdf 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java @@ -177,9 +177,7 @@ protected Object parseSourceValue(Object value) { @Override public DocValueFormat docValueFormat(String format, ZoneId timeZone) { - throw new IllegalArgumentException( - "Field [" + name() + "] of type [" + typeName() + "] doesn't support docvalue_fields or aggregations" - ); + return DocValueFormat.DENSE_VECTOR; } @Override diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml index d5fe9b3ac1e9c..bcfb9bcd79b90 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml @@ -13,6 +13,8 @@ setup: number_of_shards: 1 mappings: properties: + id: + type: keyword vector: type: rank_vectors dims: 5 @@ -29,6 +31,7 @@ setup: index: test-index id: "1" body: + id: "1" vector: [[230.0, 300.33, -34.8988, 15.555, -200.0], [-0.5, 100.0, -13, 14.8, -156.0]] byte_vector: [[8, 5, -15, 1, -7], [-1, 115, -3, 4, -128]] bit_vector: [[8, 5, -15, 1, -7], [-1, 115, -3, 4, -128]] @@ -38,6 +41,7 @@ setup: index: test-index id: "3" body: + id: "3" vector: [[0.5, 111.3, -13.0, 14.8, -156.0]] byte_vector: [[2, 18, -5, 0, -124]] bit_vector: [[2, 18, -5, 0, -124]] @@ -172,3 +176,35 @@ setup: - match: {hits.hits.1._id: "3"} - close_to: {hits.hits.1._score: {value: 2, error: 0.01}} +--- +"Test doc value vector access": + - skip: + features: close_to + + - do: + search: + _source: false + index: test-index + body: + docvalue_fields: [name, bit_vector, byte_vector, vector] + sort: [{id: "asc"}] + + - match: {hits.hits.0._id: "1"} + # vector: [[230.0, 300.33, -34.8988, 15.555, -200.0], [-0.5, 100.0, -13, 14.8, -156.0]] + - close_to: {hits.hits.0.fields.vector.0.0.0: {value: 230, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.0.1: {value: 300.33, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.0.2: {value: -34.8988, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.0.3: {value: 15.555, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.0.4: {value: -200, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.1.0: {value: -0.5, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.1.1: {value: 100, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.1.2: {value: -13, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.1.3: {value: 14.8, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.1.4: {value: -156, error: 0.0001}} + + # byte_vector: [[8, 5, -15, 1, -7], [-1, 115, -3, 4, -128]] + - match: {hits.hits.0.fields.byte_vector.0.0: [8, 5, -15, 1, -7]} + - match: {hits.hits.0.fields.bit_vector.0.0: [8, 5, -15, 1, -7]} + - match: {hits.hits.0.fields.byte_vector.0.1: [-1, 115, -3, 4, -128]} + - match: {hits.hits.0.fields.bit_vector.0.1: [-1, 115, -3, 4, -128]} + From 239564dacf755a1ccc5ee167fe7d97dec9a0ae05 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Wed, 18 Dec 2024 14:26:23 -0500 Subject: [PATCH 09/14] fixing checks --- .../xpack/rank/vectors/mapper/RankVectorsFieldMapper.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java index 4e4d4ebac8fdf..afd6fc66debbc 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java @@ -231,6 +231,9 @@ public boolean parsesArrayValue() { @Override public void parse(DocumentParserContext context) throws IOException { + if (RANK_VECTORS_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) { + throw LicenseUtils.newComplianceException("Rank Vectors"); + } if (context.doc().getByKey(fieldType().name()) != null) { throw new IllegalArgumentException( "Field [" From 5922e2a623c9fde1d5230a9e427799931faed0eb Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:58:27 -0500 Subject: [PATCH 10/14] fixing javadocs --- .../index/mapper/vectors/DenseVectorFieldMapper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 0be20babe1f25..9dfb31635901d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2512,8 +2512,9 @@ public String fieldName() { } /** - * @FunctionalInterface for a function that takes a int and boolean + * Interface for a function that takes a int and boolean */ + @FunctionalInterface public interface IntBooleanConsumer { void accept(int value, boolean isComplete); } From 399e9d79f2079fd3be2cd84671e1b23480c6c3e6 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Thu, 19 Dec 2024 12:16:31 -0500 Subject: [PATCH 11/14] trying to fix tests --- .../xpack/rank/vectors/RankVectorsPlugin.java | 18 +++++- .../mapper/RankVectorsFieldMapper.java | 59 ++++++++++++------- .../rank/vectors/LocalStateRankVectors.java | 36 +++++++++++ .../mapper/RankVectorsFieldMapperTests.java | 9 +++ .../mapper/RankVectorsFieldTypeTests.java | 12 +++- 5 files changed, 111 insertions(+), 23 deletions(-) create mode 100644 x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java index 0158e362228df..35c87f1fc1847 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java @@ -7,15 +7,22 @@ package org.elasticsearch.xpack.rank.vectors; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.license.License; +import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.LicensedFeature; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsFieldMapper; import java.util.Map; +import static org.elasticsearch.index.mapper.FieldMapper.notInMultiFields; +import static org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsFieldMapper.CONTENT_TYPE; + public class RankVectorsPlugin extends Plugin implements MapperPlugin { public static final LicensedFeature.Momentary RANK_VECTORS_FEATURE = LicensedFeature.momentary( null, @@ -25,6 +32,15 @@ public class RankVectorsPlugin extends Plugin implements MapperPlugin { @Override public Map getMappers() { - return Map.of(RankVectorsFieldMapper.CONTENT_TYPE, RankVectorsFieldMapper.PARSER); + return Map.of(CONTENT_TYPE, new FieldMapper.TypeParser((n, c) -> { + if (RANK_VECTORS_FEATURE.check(getLicenseState()) == false) { + throw LicenseUtils.newComplianceException("Rank Vectors"); + } + return new RankVectorsFieldMapper.Builder(n, c.indexVersionCreated(), getLicenseState()); + }, notInMultiFields(CONTENT_TYPE))); + } + + protected XPackLicenseState getLicenseState() { + return XPackPlugin.getSharedLicenseState(); } } diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java index afd6fc66debbc..873d67e76b04a 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java @@ -31,12 +31,12 @@ import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.vectors.VectorData; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.XPackPlugin; import java.io.IOException; import java.nio.ByteBuffer; @@ -111,10 +111,12 @@ public static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); private final IndexVersion indexCreatedVersion; + private final XPackLicenseState licenseState; - public Builder(String name, IndexVersion indexCreatedVersion) { + public Builder(String name, IndexVersion indexCreatedVersion, XPackLicenseState licenseState) { super(name); this.indexCreatedVersion = indexCreatedVersion; + this.licenseState = licenseState; } @Override @@ -125,7 +127,7 @@ protected Parameter[] getParameters() { @Override public RankVectorsFieldMapper build(MapperBuilderContext context) { // Validate on Mapping creation - if (RANK_VECTORS_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) { + if (RANK_VECTORS_FEATURE.check(licenseState) == false) { throw LicenseUtils.newComplianceException("Rank Vectors"); } // Validate again here because the dimensions or element type could have been set programmatically, @@ -133,28 +135,36 @@ public RankVectorsFieldMapper build(MapperBuilderContext context) { validate(); return new RankVectorsFieldMapper( leafName(), - new RankVectorsFieldType(context.buildFullName(leafName()), elementType.getValue(), dims.getValue(), meta.getValue()), + new RankVectorsFieldType( + context.buildFullName(leafName()), + elementType.getValue(), + dims.getValue(), + licenseState, + meta.getValue() + ), builderParams(this, context), - indexCreatedVersion + indexCreatedVersion, + licenseState ); } } - public static final TypeParser PARSER = new TypeParser((n, c) -> { - if (RANK_VECTORS_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) { - throw LicenseUtils.newComplianceException("Rank Vectors"); - } - return new Builder(n, c.indexVersionCreated()); - }, notInMultiFields(CONTENT_TYPE)); - public static final class RankVectorsFieldType extends SimpleMappedFieldType { private final DenseVectorFieldMapper.ElementType elementType; private final Integer dims; - - public RankVectorsFieldType(String name, DenseVectorFieldMapper.ElementType elementType, Integer dims, Map meta) { + private final XPackLicenseState licenseState; + + public RankVectorsFieldType( + String name, + DenseVectorFieldMapper.ElementType elementType, + Integer dims, + XPackLicenseState licenseState, + Map meta + ) { super(name, false, false, true, TextSearchInfo.NONE, meta); this.elementType = elementType; this.dims = dims; + this.licenseState = licenseState; } @Override @@ -187,7 +197,7 @@ public boolean isAggregatable() { @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { - if (RANK_VECTORS_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) { + if (RANK_VECTORS_FEATURE.check(licenseState) == false) { throw LicenseUtils.newComplianceException("Rank Vectors"); } return new RankVectorsIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, dims, elementType); @@ -213,10 +223,18 @@ DenseVectorFieldMapper.ElementType getElementType() { } private final IndexVersion indexCreatedVersion; - - private RankVectorsFieldMapper(String simpleName, MappedFieldType fieldType, BuilderParams params, IndexVersion indexCreatedVersion) { + private final XPackLicenseState licenseState; + + private RankVectorsFieldMapper( + String simpleName, + MappedFieldType fieldType, + BuilderParams params, + IndexVersion indexCreatedVersion, + XPackLicenseState licenseState + ) { super(simpleName, fieldType, params); this.indexCreatedVersion = indexCreatedVersion; + this.licenseState = licenseState; } @Override @@ -231,7 +249,7 @@ public boolean parsesArrayValue() { @Override public void parse(DocumentParserContext context) throws IOException { - if (RANK_VECTORS_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) { + if (RANK_VECTORS_FEATURE.check(licenseState) == false) { throw LicenseUtils.newComplianceException("Rank Vectors"); } if (context.doc().getByKey(fieldType().name()) != null) { @@ -267,9 +285,10 @@ public void parse(DocumentParserContext context) throws IOException { fieldType().name(), fieldType().elementType, currentDims, + licenseState, fieldType().meta() ); - Mapper update = new RankVectorsFieldMapper(leafName(), updatedFieldType, builderParams, indexCreatedVersion); + Mapper update = new RankVectorsFieldMapper(leafName(), updatedFieldType, builderParams, indexCreatedVersion, licenseState); context.addDynamicMapper(update); return; } @@ -351,7 +370,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexCreatedVersion).init(this); + return new Builder(leafName(), indexCreatedVersion, licenseState).init(this); } @Override diff --git a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java new file mode 100644 index 0000000000000..ab2f6b029cc53 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rank.vectors; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; + +import java.util.Map; + +public class LocalStateRankVectors extends LocalStateCompositeXPackPlugin { + + private final RankVectorsPlugin rankVectorsPlugin; + + public LocalStateRankVectors(Settings settings) { + super(settings, null); + LocalStateRankVectors thisVar = this; + rankVectorsPlugin = new RankVectorsPlugin() { + @Override + protected XPackLicenseState getLicenseState() { + return thisVar.getLicenseState(); + } + }; + } + + @Override + public Map getMappers() { + return rankVectorsPlugin.getMappers(); + } +} diff --git a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java index 7d9ad83eca925..25264976c58a5 100644 --- a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java @@ -27,10 +27,12 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.search.lookup.SourceProvider; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.rank.vectors.LocalStateRankVectors; import org.junit.AssumptionViolatedException; import java.io.IOException; @@ -38,6 +40,8 @@ import java.nio.ByteOrder; import java.nio.FloatBuffer; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Stream; @@ -58,6 +62,11 @@ public RankVectorsFieldMapperTests() { this.dims = ElementType.BIT == elementType ? 4 * Byte.SIZE : 4; } + @Override + protected Collection getPlugins() { + return Collections.singletonList(new LocalStateRankVectors(SETTINGS)); + } + @Override protected void minimalMapping(XContentBuilder b) throws IOException { indexMapping(b, IndexVersion.current()); diff --git a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java index 30de24939bb24..3cc2263bf19df 100644 --- a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java @@ -11,6 +11,9 @@ import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.license.License; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.license.internal.XPackLicenseStatus; import org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsFieldMapper.RankVectorsFieldType; import java.io.IOException; @@ -22,12 +25,17 @@ public class RankVectorsFieldTypeTests extends FieldTypeTestCase { + private final XPackLicenseState licenseState = new XPackLicenseState( + System::currentTimeMillis, + new XPackLicenseStatus(License.OperationMode.TRIAL, true, null) + ); + private RankVectorsFieldType createFloatFieldType() { - return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.FLOAT, BBQ_MIN_DIMS, Collections.emptyMap()); + return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.FLOAT, BBQ_MIN_DIMS, licenseState, Collections.emptyMap()); } private RankVectorsFieldMapper.RankVectorsFieldType createByteFieldType() { - return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.BYTE, 5, Collections.emptyMap()); + return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.BYTE, 5, licenseState, Collections.emptyMap()); } public void testHasDocValues() { From e9ad982fe3d52def398de4c40b39c2c54ce97bde Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 6 Jan 2025 08:22:34 -0500 Subject: [PATCH 12/14] fixing tests --- .../mapping/types/rank-vectors.asciidoc | 26 ++++++++++++++++--- .../rank/vectors/LocalStateRankVectors.java | 8 +++++- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/docs/reference/mapping/types/rank-vectors.asciidoc b/docs/reference/mapping/types/rank-vectors.asciidoc index 14668c03c5bd6..a718a5e47ec85 100644 --- a/docs/reference/mapping/types/rank-vectors.asciidoc +++ b/docs/reference/mapping/types/rank-vectors.asciidoc @@ -32,6 +32,7 @@ PUT my-rank-vectors-float/_doc/1 } -------------------------------------------------- +// TESTSETUP In addition to the `float` element type, `byte` and `bit` element types are also supported. @@ -73,14 +74,13 @@ PUT my-rank-vectors-bit } } -POST /my-bit-vectors/_bulk?refresh +POST /my-rank-vectors-bit/_bulk?refresh {"index": {"_id" : "1"}} {"my_vector": [127, -127, 0, 1, 42]} {"index": {"_id" : "2"}} {"my_vector": "8100012a7f"} -------------------------------------------------- - [role="child_attributes"] [[rank-vectors-params]] ==== Parameters for rank vectors fields @@ -135,7 +135,7 @@ For example, the following query scores documents based on the maxSim similarity [source,console] -------------------------------------------------- -GET my-index/_search +GET my-rank-vectors-float/_search { "query": { "script_score": { @@ -157,7 +157,25 @@ Additionally, asymmetric similarity functions can be used to score against `bit` [source,console] -------------------------------------------------- -GET my-index/_search +PUT my-rank-vectors-bit +{ + "mappings": { + "properties": { + "my_vector": { + "type": "rank_vectors", + "element_type": "bit" + } + } + } +} + +POST /my-rank-vectors-bit/_bulk?refresh +{"index": {"_id" : "1"}} +{"my_vector": [127, -127, 0, 1, 42]} +{"index": {"_id" : "2"}} +{"my_vector": "8100012a7f"} + +GET my-rank-vectors-bit/_search { "query": { "script_score": { diff --git a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java index ab2f6b029cc53..ba085b8f0d705 100644 --- a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java @@ -9,7 +9,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.license.License; import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.license.internal.XPackLicenseStatus; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import java.util.Map; @@ -17,6 +19,10 @@ public class LocalStateRankVectors extends LocalStateCompositeXPackPlugin { private final RankVectorsPlugin rankVectorsPlugin; + private final XPackLicenseState licenseState = new XPackLicenseState( + System::currentTimeMillis, + new XPackLicenseStatus(License.OperationMode.TRIAL, true, null) + ); public LocalStateRankVectors(Settings settings) { super(settings, null); @@ -24,7 +30,7 @@ public LocalStateRankVectors(Settings settings) { rankVectorsPlugin = new RankVectorsPlugin() { @Override protected XPackLicenseState getLicenseState() { - return thisVar.getLicenseState(); + return licenseState; } }; } From 99276adc2e5b695da300c369e81d92d0c92b5974 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 6 Jan 2025 09:44:49 -0500 Subject: [PATCH 13/14] fixing gradle --- x-pack/plugin/rank-vectors/build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/rank-vectors/build.gradle b/x-pack/plugin/rank-vectors/build.gradle index cd550d96a5503..53aabb8fdbf74 100644 --- a/x-pack/plugin/rank-vectors/build.gradle +++ b/x-pack/plugin/rank-vectors/build.gradle @@ -9,9 +9,9 @@ apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - name 'rank-vectors' - description 'Rank vectors in search.' - classname 'org.elasticsearch.xpack.rank.vectors.RankVectorsPlugin' + name = 'rank-vectors' + description = 'Rank vectors in search.' + classname = 'org.elasticsearch.xpack.rank.vectors.RankVectorsPlugin' extendedPlugins = ['x-pack-core', 'lang-painless'] } From ca4a8a2cad64c2c06b4ab94fbc1a39e32c8f7e24 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 6 Jan 2025 11:06:26 -0500 Subject: [PATCH 14/14] fixing tests --- .../vectors/script/RankVectorsScoreScriptUtils.java | 10 ---------- .../rank/vectors/mapper/RankVectorsFieldTypeTests.java | 5 +++-- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java index 722344d7cfe0a..3d692db08ff9e 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java @@ -9,18 +9,14 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; -import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.field.vectors.DenseVector; import org.elasticsearch.script.field.vectors.RankVectorsDocValuesField; -import org.elasticsearch.xpack.core.XPackPlugin; import java.io.IOException; import java.util.HexFormat; import java.util.List; -import static org.elasticsearch.xpack.rank.vectors.RankVectorsPlugin.RANK_VECTORS_FEATURE; - public class RankVectorsScoreScriptUtils { public static class RankVectorsFunction { @@ -182,9 +178,6 @@ public static final class MaxSimInvHamming { private final MaxSimInvHammingDistanceInterface function; public MaxSimInvHamming(ScoreScript scoreScript, Object queryVector, String fieldName) { - if (RANK_VECTORS_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) { - throw LicenseUtils.newComplianceException("Rank Vectors"); - } RankVectorsDocValuesField field = (RankVectorsDocValuesField) scoreScript.field(fieldName); if (field.getElementType() == DenseVectorFieldMapper.ElementType.FLOAT) { throw new IllegalArgumentException("hamming distance is only supported for byte or bit vectors"); @@ -340,9 +333,6 @@ public static final class MaxSimDotProduct { @SuppressWarnings("unchecked") public MaxSimDotProduct(ScoreScript scoreScript, Object queryVector, String fieldName) { - if (RANK_VECTORS_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) { - throw LicenseUtils.newComplianceException("Rank Vectors"); - } RankVectorsDocValuesField field = (RankVectorsDocValuesField) scoreScript.field(fieldName); function = switch (field.getElementType()) { case BIT -> { diff --git a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java index 3cc2263bf19df..59c5b0414910d 100644 --- a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.license.License; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.license.internal.XPackLicenseStatus; +import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsFieldMapper.RankVectorsFieldType; import java.io.IOException; @@ -78,9 +79,9 @@ public void testFielddataBuilder() { public void testDocValueFormat() { RankVectorsFieldType fft = createFloatFieldType(); - expectThrows(IllegalArgumentException.class, () -> fft.docValueFormat(null, null)); + assertEquals(DocValueFormat.DENSE_VECTOR, fft.docValueFormat(null, null)); RankVectorsFieldType bft = createByteFieldType(); - expectThrows(IllegalArgumentException.class, () -> bft.docValueFormat(null, null)); + assertEquals(DocValueFormat.DENSE_VECTOR, bft.docValueFormat(null, null)); } public void testFetchSourceValue() throws IOException {