From 9147de61948a371c83a4562130a404bca43052df Mon Sep 17 00:00:00 2001 From: John Wagster Date: Thu, 28 Aug 2025 14:25:24 -0500 Subject: [PATCH 01/36] pulled new optional param visit_percentage through the query logic --- .../percolator/PercolatorQuerySearchIT.java | 2 +- .../DenseVectorFieldIndexTypeUpdateIT.java | 2 +- .../org/elasticsearch/search/KnnSearchIT.java | 8 +-- .../search/query/RescoreKnnVectorQueryIT.java | 4 ++ .../retriever/RetrieverTelemetryIT.java | 4 +- .../org/elasticsearch/TransportVersions.java | 1 + .../vectors/DenseVectorFieldMapper.java | 12 +++- .../search/vectors/KnnSearchBuilder.java | 42 ++++++++++++-- .../vectors/KnnSearchRequestParser.java | 12 +++- .../search/vectors/KnnVectorQueryBuilder.java | 45 ++++++++++++--- .../search/KnnSearchSingleNodeTests.java | 22 ++++---- .../vectors/DenseVectorFieldMapperTests.java | 10 ++++ .../index/query/NestedQueryBuilderTests.java | 1 + ...AbstractKnnVectorQueryBuilderTestCase.java | 46 +++++++++++++-- .../KnnByteVectorQueryBuilderTests.java | 3 +- .../KnnFloatVectorQueryBuilderTests.java | 3 +- .../search/vectors/KnnSearchBuilderTests.java | 56 ++++++++++++++++--- .../vectors/KnnSearchRequestParserTests.java | 39 ++++++++++++- .../xpack/esql/querydsl/query/KnnQuery.java | 5 +- .../LocalPhysicalPlanOptimizerTests.java | 19 +++++-- .../mapper/SemanticTextFieldMapper.java | 2 +- ...anticKnnVectorQueryRewriteInterceptor.java | 4 ++ ...KnnVectorQueryRewriteInterceptorTests.java | 6 +- .../SemanticTextHighlighterTests.java | 3 +- ...SimilarityRankRetrieverTelemetryTests.java | 4 +- .../xpack/rank/linear/LinearRetrieverIT.java | 2 +- .../xpack/rank/rrf/RRFRetrieverBuilderIT.java | 2 +- .../rank/rrf/RRFRetrieverTelemetryIT.java | 4 +- .../DocumentLevelSecurityTests.java | 2 +- .../integration/FieldLevelSecurityTests.java | 6 +- .../upgrades/SemanticTextUpgradeIT.java | 2 +- 31 files changed, 297 insertions(+), 76 deletions(-) diff --git a/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java index 8a7f1405f8f4e..3aaf668851ded 100644 --- a/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java @@ -1359,7 +1359,7 @@ public void testKnnQueryNotSupportedInPercolator() throws IOException { """); indicesAdmin().prepareCreate("index1").setMapping(mappings).get(); ensureGreen(); - QueryBuilder knnVectorQueryBuilder = new KnnVectorQueryBuilder("my_vector", new float[] { 1, 1, 1, 1, 1 }, 10, 10, null, null); + QueryBuilder knnVectorQueryBuilder = new KnnVectorQueryBuilder("my_vector", new float[] { 1, 1, 1, 1, 1 }, 10, 10, 10f,null, null); IndexRequestBuilder indexRequestBuilder = prepareIndex("index1").setId("knn_query1") .setSource(jsonBuilder().startObject().field("my_query", knnVectorQueryBuilder).endObject()); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldIndexTypeUpdateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldIndexTypeUpdateIT.java index de174774f980a..6066d53605c99 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldIndexTypeUpdateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldIndexTypeUpdateIT.java @@ -142,7 +142,7 @@ public void testDenseVectorMappingUpdate() throws Exception { for (int i = 0; i < queryVector.length; i++) { queryVector[i] = randomFloatBetween(-1, 1, true); } - KnnVectorQueryBuilder queryBuilder = new KnnVectorQueryBuilder(VECTOR_FIELD, queryVector, null, null, null, null); + KnnVectorQueryBuilder queryBuilder = new KnnVectorQueryBuilder(VECTOR_FIELD, queryVector, null, null, null,null, null); assertNoFailuresAndResponse( client().prepareSearch(INDEX_NAME).setQuery(queryBuilder).setTrackTotalHits(true).setSize(expectedDocs), response -> { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/KnnSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/KnnSearchIT.java index 91409e5e70183..17d6024145e22 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/KnnSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/KnnSearchIT.java @@ -77,13 +77,13 @@ public void testKnnSearchWithScroll() throws Exception { // test top level knn search { SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - sourceBuilder.knnSearch(List.of(new KnnSearchBuilder(VECTOR_FIELD, new float[] { 0, 0 }, k, 100, null, null))); + sourceBuilder.knnSearch(List.of(new KnnSearchBuilder(VECTOR_FIELD, new float[] { 0, 0 }, k, 100, 10f, null, null))); executeScrollSearch(client, sourceBuilder, k); } // test top level knn search + another query { SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - sourceBuilder.knnSearch(List.of(new KnnSearchBuilder(VECTOR_FIELD, new float[] { 0, 0 }, k, 100, null, null))); + sourceBuilder.knnSearch(List.of(new KnnSearchBuilder(VECTOR_FIELD, new float[] { 0, 0 }, k, 100, 10f, null, null))); sourceBuilder.query(QueryBuilders.existsQuery("category").boost(10)); executeScrollSearch(client, sourceBuilder, k + 10); } @@ -91,7 +91,7 @@ public void testKnnSearchWithScroll() throws Exception { // test knn query { SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - sourceBuilder.query(new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 0, 0 }, k, 100, null, null)); + sourceBuilder.query(new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 0, 0 }, k, 100, 10f, null, null)); executeScrollSearch(client, sourceBuilder, k * numShards); } // test knn query + another query @@ -99,7 +99,7 @@ public void testKnnSearchWithScroll() throws Exception { SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.query( QueryBuilders.boolQuery() - .should(new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 0, 0 }, k, 100, null, null)) + .should(new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 0, 0 }, k, 100, 10f, null, null)) .should(QueryBuilders.existsQuery("category").boost(10)) ); executeScrollSearch(client, sourceBuilder, k * numShards + 10); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java index 453812c0566f7..43aa2c9effb1f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java @@ -116,6 +116,7 @@ private record TestParams( float[] queryVector, int k, int numCands, + float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder ) { public static TestParams generate() { @@ -128,6 +129,7 @@ public static TestParams generate() { randomVector(numDims), k, (int) (k * randomFloatBetween(1.0f, 10.0f, true)), + (float) randomFloatBetween(0.0f, 100.0f, true), new RescoreVectorBuilder(randomFloatBetween(1.0f, 100f, true)) ); } @@ -140,6 +142,7 @@ public void testKnnSearchRescore() { testParams.queryVector, testParams.k, testParams.numCands, + testParams.visitPercentage, testParams.rescoreVectorBuilder, null ); @@ -155,6 +158,7 @@ public void testKnnQueryRescore() { testParams.queryVector, testParams.k, testParams.numCands, + testParams.visitPercentage, testParams.rescoreVectorBuilder, null ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverTelemetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverTelemetryIT.java index 1762e4fe299c4..8256d19f12d84 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverTelemetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverTelemetryIT.java @@ -99,7 +99,7 @@ public void testTelemetryForRetrievers() throws IOException { { performSearch( new SearchSourceBuilder().retriever( - new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", new float[] { 1.0f }, 10, 15, null, null)) + new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", new float[] { 1.0f }, 10, 15, 10f,null, null)) ) ); } @@ -114,7 +114,7 @@ public void testTelemetryForRetrievers() throws IOException { // his will record 1 entry for "knn" in `sections` { performSearch( - new SearchSourceBuilder().knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 1.0f }, 10, 15, null, null))) + new SearchSourceBuilder().knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 1.0f }, 10, 15, 10f, null, null))) ); } diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index da61d44264571..f82440f620d89 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -224,6 +224,7 @@ static TransportVersion def(int id) { public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_4 = def(9_000_0_13); public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_6 = def(9_000_0_15); public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_7 = def(9_000_0_16); + public static final TransportVersion V_9_1_0 = def(9_001_0_00); public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_001_0_00); public static final TransportVersion REMOVE_SNAPSHOT_FAILURES = def(9_002_0_00); public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED = def(9_003_0_00); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 9c5d28a4942d3..22abb1c4eca0e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2530,6 +2530,7 @@ public Query createKnnQuery( VectorData queryVector, int k, int numCands, + float visitPercentage, Float oversample, Query filter, Float similarityThreshold, @@ -2551,6 +2552,7 @@ public Query createKnnQuery( queryVector.asByteVector(), k, numCands, + visitPercentage, filter, similarityThreshold, parentFilter, @@ -2561,6 +2563,7 @@ public Query createKnnQuery( queryVector.asFloatVector(), k, numCands, + visitPercentage, oversample, filter, similarityThreshold, @@ -2572,6 +2575,7 @@ public Query createKnnQuery( queryVector.asByteVector(), k, numCands, + visitPercentage, filter, similarityThreshold, parentFilter, @@ -2593,6 +2597,7 @@ private Query createKnnBitQuery( byte[] queryVector, int k, int numCands, + float visitPercentage, Query filter, Float similarityThreshold, BitSetProducer parentFilter, @@ -2632,6 +2637,7 @@ private Query createKnnByteQuery( byte[] queryVector, int k, int numCands, + float visitPercentage, Query filter, Float similarityThreshold, BitSetProducer parentFilter, @@ -2693,6 +2699,7 @@ private Query createKnnFloatQuery( float[] queryVector, int k, int numCands, + float visitPercentage, Float queryOversample, Query filter, Float similarityThreshold, @@ -2741,6 +2748,7 @@ private Query createKnnFloatQuery( .build(); } else if (indexOptions instanceof BBQIVFIndexOptions bbqIndexOptions) { float defaultVisitRatio = (float) (bbqIndexOptions.defaultVisitPercentage / 100d); + float visitRatio = visitPercentage == 0.0f ? defaultVisitRatio : (float) (visitPercentage / 100d); knnQuery = parentFilter != null ? new DiversifyingChildrenIVFKnnFloatVectorQuery( name(), @@ -2749,9 +2757,9 @@ private Query createKnnFloatQuery( numCands, filter, parentFilter, - defaultVisitRatio + visitRatio ) - : new IVFKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter, defaultVisitRatio); + : new IVFKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter, visitRatio); } else { knnQuery = parentFilter != null ? new ESDiversifyingChildrenFloatKnnVectorQuery( diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index 9b9718efcf523..cf83e1df20f68 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -118,6 +118,7 @@ public static KnnSearchBuilder.Builder fromXContent(XContentParser parser) throw private final Supplier querySupplier; final int k; final int numCands; + final float visitPercentage; final Float similarity; final List filterQueries; String queryName; @@ -139,6 +140,7 @@ public KnnSearchBuilder( float[] queryVector, int k, int numCands, + float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity ) { @@ -148,6 +150,7 @@ public KnnSearchBuilder( null, k, numCands, + visitPercentage, rescoreVectorBuilder, similarity ); @@ -166,10 +169,11 @@ public KnnSearchBuilder( VectorData queryVector, int k, int numCands, + float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity ) { - this(field, queryVector, null, k, numCands, rescoreVectorBuilder, similarity); + this(field, queryVector, null, k, numCands, visitPercentage, rescoreVectorBuilder, similarity); } /** @@ -185,6 +189,7 @@ public KnnSearchBuilder( QueryVectorBuilder queryVectorBuilder, int k, int numCands, + float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity ) { @@ -194,6 +199,7 @@ public KnnSearchBuilder( Objects.requireNonNull(queryVectorBuilder, format("[%s] cannot be null", QUERY_VECTOR_BUILDER_FIELD.getPreferredName())), k, numCands, + visitPercentage, rescoreVectorBuilder, similarity ); @@ -205,6 +211,7 @@ public KnnSearchBuilder( QueryVectorBuilder queryVectorBuilder, int k, int numCands, + float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity ) { @@ -215,6 +222,7 @@ public KnnSearchBuilder( new ArrayList<>(), k, numCands, + visitPercentage, rescoreVectorBuilder, similarity, null, @@ -228,6 +236,7 @@ private KnnSearchBuilder( Supplier querySupplier, Integer k, Integer numCands, + Float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, List filterQueries, Float similarity @@ -237,6 +246,7 @@ private KnnSearchBuilder( this.queryVectorBuilder = null; this.k = k; this.numCands = numCands; + this.visitPercentage = visitPercentage; this.filterQueries = filterQueries; this.querySupplier = querySupplier; this.similarity = similarity; @@ -250,6 +260,7 @@ private KnnSearchBuilder( List filterQueries, int k, int numCandidates, + float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity, InnerHitBuilder innerHitBuilder, @@ -267,6 +278,9 @@ private KnnSearchBuilder( if (numCandidates > NUM_CANDS_LIMIT) { throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot exceed [" + NUM_CANDS_LIMIT + "]"); } + if (visitPercentage < 0f || visitPercentage > 100f) { + throw new IllegalArgumentException("[" + "visit_percentage" + "] must be between [0] and [100]"); + } if (queryVector == null && queryVectorBuilder == null) { throw new IllegalArgumentException( format( @@ -290,6 +304,7 @@ private KnnSearchBuilder( this.queryVectorBuilder = queryVectorBuilder; this.k = k; this.numCands = numCandidates; + this.visitPercentage = visitPercentage; this.rescoreVectorBuilder = rescoreVectorBuilder; this.innerHitBuilder = innerHitBuilder; this.similarity = similarity; @@ -303,6 +318,11 @@ public KnnSearchBuilder(StreamInput in) throws IOException { this.field = in.readString(); this.k = in.readVInt(); this.numCands = in.readVInt(); + if (in.getTransportVersion().onOrAfter(TransportVersions.V_9_1_0)) { + this.visitPercentage = in.readFloat(); + } else { + this.visitPercentage = 0f; + } if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { this.queryVector = in.readOptionalWriteable(VectorData::new); } else { @@ -416,7 +436,8 @@ public KnnSearchBuilder rewrite(QueryRewriteContext ctx) throws IOException { if (querySupplier.get() == null) { return this; } - return new KnnSearchBuilder(field, querySupplier.get(), k, numCands, rescoreVectorBuilder, similarity).boost(boost) + return new KnnSearchBuilder(field, querySupplier.get(), k, numCands, + visitPercentage, rescoreVectorBuilder, similarity).boost(boost) .queryName(queryName) .addFilterQueries(filterQueries) .innerHit(innerHitBuilder); @@ -439,7 +460,8 @@ public KnnSearchBuilder rewrite(QueryRewriteContext ctx) throws IOException { } ll.onResponse(null); }))); - return new KnnSearchBuilder(field, toSet::get, k, numCands, rescoreVectorBuilder, filterQueries, similarity).boost(boost) + return new KnnSearchBuilder(field, toSet::get, k, numCands, + visitPercentage, rescoreVectorBuilder, filterQueries, similarity).boost(boost) .queryName(queryName) .innerHit(innerHitBuilder); } @@ -453,7 +475,7 @@ public KnnSearchBuilder rewrite(QueryRewriteContext ctx) throws IOException { rewrittenQueries.add(rewrittenQuery); } if (changed) { - return new KnnSearchBuilder(field, queryVector, k, numCands, rescoreVectorBuilder, similarity).boost(boost) + return new KnnSearchBuilder(field, queryVector, k, numCands, visitPercentage, rescoreVectorBuilder, similarity).boost(boost) .queryName(queryName) .addFilterQueries(rewrittenQueries) .innerHit(innerHitBuilder); @@ -465,7 +487,8 @@ public KnnVectorQueryBuilder toQueryBuilder() { if (queryVectorBuilder != null) { throw new IllegalArgumentException("missing rewrite"); } - return new KnnVectorQueryBuilder(field, queryVector, numCands, numCands, rescoreVectorBuilder, similarity).boost(boost) + return new KnnVectorQueryBuilder(field, queryVector, numCands, numCands, + visitPercentage, rescoreVectorBuilder, similarity).boost(boost) .queryName(queryName) .addFilterQueries(filterQueries); } @@ -601,6 +624,7 @@ public static class Builder { private QueryVectorBuilder queryVectorBuilder; private Integer k; private Integer numCandidates; + private Float visitPercentage; private Float similarity; private final List filterQueries = new ArrayList<>(); private String queryName; @@ -654,6 +678,11 @@ public Builder numCandidates(Integer numCands) { return this; } + public Builder visitPercentage(Float visitPercentage) { + this.visitPercentage = visitPercentage; + return this; + } + public Builder similarity(Float similarity) { this.similarity = similarity; return this; @@ -670,6 +699,8 @@ public KnnSearchBuilder build(int size) { int adjustedNumCandidates = numCandidates == null ? Math.round(Math.min(NUM_CANDS_LIMIT, NUM_CANDS_MULTIPLICATIVE_FACTOR * adjustedK)) : numCandidates; + // FIXME: adjust this based on the other params? + float adjustedVisitPercentage = visitPercentage == null ? 0f : visitPercentage; return new KnnSearchBuilder( field, queryVectorBuilder, @@ -677,6 +708,7 @@ public KnnSearchBuilder build(int size) { filterQueries, adjustedK, adjustedNumCandidates, + adjustedVisitPercentage, rescoreVectorBuilder, similarity, innerHitBuilder, diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java index 12573d5ad496e..5ab90c570f57a 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java @@ -199,6 +199,7 @@ static class KnnSearch { static final ParseField FIELD_FIELD = new ParseField("field"); static final ParseField K_FIELD = new ParseField("k"); static final ParseField NUM_CANDS_FIELD = new ParseField("num_candidates"); + static final ParseField VISIT_PERCENTAGE_FIELD = new ParseField("visit_percentage"); static final ParseField QUERY_VECTOR_FIELD = new ParseField("query_vector"); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("knn", args -> { @@ -208,7 +209,7 @@ static class KnnSearch { for (int i = 0; i < vector.size(); i++) { vectorArray[i] = vector.get(i); } - return new KnnSearch((String) args[0], vectorArray, (int) args[2], (int) args[3]); + return new KnnSearch((String) args[0], vectorArray, (int) args[2], (int) args[3], (float) args[4]); }); static { @@ -226,6 +227,7 @@ public static KnnSearch parse(XContentParser parser) throws IOException { final float[] queryVector; final int k; final int numCands; + final float visitPercentage; /** * Defines a kNN search. @@ -235,11 +237,12 @@ public static KnnSearch parse(XContentParser parser) throws IOException { * @param k the final number of nearest neighbors to return as top hits * @param numCands the number of nearest neighbor candidates to consider per shard */ - KnnSearch(String field, float[] queryVector, int k, int numCands) { + KnnSearch(String field, float[] queryVector, int k, int numCands, float visitPercentage) { this.field = field; this.queryVector = queryVector; this.k = k; this.numCands = numCands; + this.visitPercentage = visitPercentage; } public KnnVectorQueryBuilder toQueryBuilder() { @@ -256,7 +259,10 @@ public KnnVectorQueryBuilder toQueryBuilder() { if (numCands > NUM_CANDS_LIMIT) { throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot exceed [" + NUM_CANDS_LIMIT + "]"); } - return new KnnVectorQueryBuilder(field, queryVector, numCands, numCands, null, null); + if(visitPercentage < 0.0f || visitPercentage > 100.0f) { + throw new IllegalArgumentException("[" + VISIT_PERCENTAGE_FIELD.getPreferredName() + "] must be between 0 and 100"); + } + return new KnnVectorQueryBuilder(field, queryVector, numCands, numCands, visitPercentage, null, null); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index b76f56ceb2aa9..77abdefd98d4a 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -64,6 +64,7 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder new KnnVectorQueryBuilder( (String) args[0], (VectorData) args[1], - (QueryVectorBuilder) args[5], + (QueryVectorBuilder) args[6], null, (Integer) args[2], (Integer) args[3], - (RescoreVectorBuilder) args[6], - (Float) args[4] + (Float) args[4], + (RescoreVectorBuilder) args[7], + (Float) args[5] ) ); @@ -123,6 +125,7 @@ public static KnnVectorQueryBuilder fromXContent(XContentParser parser) { private final VectorData queryVector; private final Integer k; private final Integer numCands; + private final Float visitPercentage; private final List filterQueries = new ArrayList<>(); private final Float vectorSimilarity; private final QueryVectorBuilder queryVectorBuilder; @@ -134,10 +137,12 @@ public KnnVectorQueryBuilder( float[] queryVector, Integer k, Integer numCands, + Float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float vectorSimilarity ) { - this(fieldName, VectorData.fromFloats(queryVector), null, null, k, numCands, rescoreVectorBuilder, vectorSimilarity); + this(fieldName, VectorData.fromFloats(queryVector), null, null, k, numCands, + visitPercentage, rescoreVectorBuilder, vectorSimilarity); } public KnnVectorQueryBuilder( @@ -145,9 +150,10 @@ public KnnVectorQueryBuilder( QueryVectorBuilder queryVectorBuilder, Integer k, Integer numCands, + Float visitPercentage, Float vectorSimilarity ) { - this(fieldName, null, queryVectorBuilder, null, k, numCands, null, vectorSimilarity); + this(fieldName, null, queryVectorBuilder, null, k, numCands, visitPercentage, null, vectorSimilarity); } public KnnVectorQueryBuilder( @@ -155,10 +161,12 @@ public KnnVectorQueryBuilder( byte[] queryVector, Integer k, Integer numCands, + Float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float vectorSimilarity ) { - this(fieldName, VectorData.fromBytes(queryVector), null, null, k, numCands, rescoreVectorBuilder, vectorSimilarity); + this(fieldName, VectorData.fromBytes(queryVector), null, null, k, numCands, + visitPercentage, rescoreVectorBuilder, vectorSimilarity); } public KnnVectorQueryBuilder( @@ -166,10 +174,11 @@ public KnnVectorQueryBuilder( VectorData queryVector, Integer k, Integer numCands, + Float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float vectorSimilarity ) { - this(fieldName, queryVector, null, null, k, numCands, rescoreVectorBuilder, vectorSimilarity); + this(fieldName, queryVector, null, null, k, numCands, visitPercentage, rescoreVectorBuilder, vectorSimilarity); } private KnnVectorQueryBuilder( @@ -179,6 +188,7 @@ private KnnVectorQueryBuilder( Supplier queryVectorSupplier, Integer k, Integer numCands, + Float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float vectorSimilarity ) { @@ -214,6 +224,7 @@ private KnnVectorQueryBuilder( this.queryVector = queryVector; this.k = k; this.numCands = numCands; + this.visitPercentage = visitPercentage; this.vectorSimilarity = vectorSimilarity; this.queryVectorBuilder = queryVectorBuilder; this.queryVectorSupplier = queryVectorSupplier; @@ -233,6 +244,12 @@ public KnnVectorQueryBuilder(StreamInput in) throws IOException { } else { this.numCands = in.readVInt(); } + // FIXME: validate transport version changes + if(in.getTransportVersion().onOrAfter(TransportVersions.V_9_1_0)) { + this.visitPercentage = in.readOptionalFloat(); + } else { + this.visitPercentage = in.readFloat(); + } if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { this.queryVector = in.readOptionalWriteable(VectorData::new); } else { @@ -289,6 +306,10 @@ public Integer numCands() { return numCands; } + public Float visitPercentage() { + return visitPercentage; + } + public List filterQueries() { return filterQueries; } @@ -422,7 +443,8 @@ protected QueryBuilder doRewrite(QueryRewriteContext ctx) throws IOException { if (queryVectorSupplier.get() == null) { return this; } - return new KnnVectorQueryBuilder(fieldName, queryVectorSupplier.get(), k, numCands, rescoreVectorBuilder, vectorSimilarity) + return new KnnVectorQueryBuilder(fieldName, queryVectorSupplier.get(), k, numCands, + visitPercentage, rescoreVectorBuilder, vectorSimilarity) .boost(boost) .queryName(queryName) .addFilterQueries(filterQueries); @@ -452,6 +474,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext ctx) throws IOException { toSet::get, k, numCands, + visitPercentage, rescoreVectorBuilder, vectorSimilarity ).boost(boost).queryName(queryName).addFilterQueries(filterQueries); @@ -476,6 +499,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext ctx) throws IOException { queryVectorSupplier, k, numCands, + visitPercentage, rescoreVectorBuilder, vectorSimilarity ).boost(boost).queryName(queryName).addFilterQueries(rewrittenQueries); @@ -512,6 +536,10 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { } } int adjustedNumCands = numCands == null ? Math.round(Math.min(NUM_CANDS_MULTIPLICATIVE_FACTOR * k, NUM_CANDS_LIMIT)) : numCands; + + // FIXME: how do the other params interact with this? + float adjustedVisitPercentage = visitPercentage == null ? 0.0f : visitPercentage; + if (fieldType == null) { return new MatchNoDocsQuery(); } @@ -575,6 +603,7 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { queryVector, k, adjustedNumCands, + adjustedVisitPercentage, oversample, filterQuery, vectorSimilarity, diff --git a/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java b/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java index 7c5ca2d9007b6..0114d1994caf9 100644 --- a/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java @@ -63,7 +63,7 @@ public void testKnnSearchRemovedVector() throws IOException { client().prepareUpdate("index", "0").setDoc("vector", (Object) null).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); float[] queryVector = randomVector(); - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 20, 50, null, null).boost(5.0f); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 20, 50, 10f, null, null).boost(5.0f); assertResponse( client().prepareSearch("index") .setKnnSearch(List.of(knnSearch)) @@ -107,7 +107,7 @@ public void testKnnWithQuery() throws IOException { indicesAdmin().prepareRefresh("index").get(); float[] queryVector = randomVector(); - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50, null, null).boost(5.0f).queryName("knn"); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50, 10f, null, null).boost(5.0f).queryName("knn"); assertResponse( client().prepareSearch("index") .setKnnSearch(List.of(knnSearch)) @@ -156,7 +156,7 @@ public void testKnnFilter() throws IOException { indicesAdmin().prepareRefresh("index").get(); float[] queryVector = randomVector(); - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50, null, null).addFilterQuery( + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50, 10f, null, null).addFilterQuery( QueryBuilders.termsQuery("field", "second") ); assertResponse(client().prepareSearch("index").setKnnSearch(List.of(knnSearch)).addFetchField("*").setSize(10), response -> { @@ -199,7 +199,7 @@ public void testKnnFilterWithRewrite() throws IOException { indicesAdmin().prepareRefresh("index").get(); float[] queryVector = randomVector(); - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50, null, null).addFilterQuery( + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50, 10f, null, null).addFilterQuery( QueryBuilders.termsLookupQuery("field", new TermsLookup("index", "lookup-doc", "other-field")) ); assertResponse(client().prepareSearch("index").setKnnSearch(List.of(knnSearch)).setSize(10), response -> { @@ -246,8 +246,8 @@ public void testMultiKnnClauses() throws IOException { indicesAdmin().prepareRefresh("index").get(); float[] queryVector = randomVector(20f, 21f); - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50, null, null).boost(5.0f); - KnnSearchBuilder knnSearch2 = new KnnSearchBuilder("vector_2", queryVector, 5, 50, null, null).boost(10.0f); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50, 10f, null, null).boost(5.0f); + KnnSearchBuilder knnSearch2 = new KnnSearchBuilder("vector_2", queryVector, 5, 50, 10f, null, null).boost(10.0f); assertResponse( client().prepareSearch("index") .setKnnSearch(List.of(knnSearch, knnSearch2)) @@ -308,8 +308,8 @@ public void testMultiKnnClausesSameDoc() throws IOException { float[] queryVector = randomVector(); // Having the same query vector and same docs should mean our KNN scores are linearly combined if the same doc is matched - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50, null, null); - KnnSearchBuilder knnSearch2 = new KnnSearchBuilder("vector_2", queryVector, 5, 50, null, null); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50, 10f, null, null); + KnnSearchBuilder knnSearch2 = new KnnSearchBuilder("vector_2", queryVector, 5, 50, 10f, null, null); assertResponse( client().prepareSearch("index") .setKnnSearch(List.of(knnSearch)) @@ -383,7 +383,7 @@ public void testKnnFilteredAlias() throws IOException { indicesAdmin().prepareRefresh("index").get(); float[] queryVector = randomVector(); - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 10, 50, null, null); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 10, 50, 10f, null, null); final int expectedHitCount = expectedHits; assertResponse(client().prepareSearch("test-alias").setKnnSearch(List.of(knnSearch)).setSize(10), response -> { assertHitCount(response, expectedHitCount); @@ -420,7 +420,7 @@ public void testKnnSearchAction() throws IOException { float[] queryVector = randomVector(); assertResponse( client().prepareSearch("index1", "index2") - .setQuery(new KnnVectorQueryBuilder("vector", queryVector, 5, 5, null, null)) + .setQuery(new KnnVectorQueryBuilder("vector", queryVector, 5, 5, 10f, null, null)) .setSize(2), response -> { // The total hits is num_cands * num_shards, since the query gathers num_cands hits from each shard @@ -454,7 +454,7 @@ public void testKnnVectorsWith4096Dims() throws IOException { indicesAdmin().prepareRefresh("index").get(); float[] queryVector = randomVector(4096); - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 3, 50, null, null).boost(5.0f); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 3, 50, 10f, null, null).boost(5.0f); assertResponse(client().prepareSearch("index").setKnnSearch(List.of(knnSearch)).addFetchField("*").setSize(10), response -> { assertHitCount(response, 3); assertEquals(3, response.getHits().getHits().length); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 00c9cb4e68ae8..11ffe2f9dc789 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -2555,6 +2555,7 @@ public void testByteVectorQueryBoundaries() throws IOException { VectorData.fromFloats(new float[] { 128, 0, 0 }), 3, 3, + 10f, null, null, null, @@ -2574,6 +2575,7 @@ public void testByteVectorQueryBoundaries() throws IOException { VectorData.fromFloats(new float[] { 0.0f, 0f, -129.0f }), 3, 3, + 10f, null, null, null, @@ -2593,6 +2595,7 @@ public void testByteVectorQueryBoundaries() throws IOException { VectorData.fromFloats(new float[] { 0.0f, 0.5f, 0.0f }), 3, 3, + 10f, null, null, null, @@ -2612,6 +2615,7 @@ public void testByteVectorQueryBoundaries() throws IOException { VectorData.fromFloats(new float[] { 0, 0.0f, -0.25f }), 3, 3, + 10f, null, null, null, @@ -2631,6 +2635,7 @@ public void testByteVectorQueryBoundaries() throws IOException { VectorData.fromFloats(new float[] { Float.NaN, 0f, 0.0f }), 3, 3, + 10f, null, null, null, @@ -2647,6 +2652,7 @@ public void testByteVectorQueryBoundaries() throws IOException { VectorData.fromFloats(new float[] { Float.POSITIVE_INFINITY, 0f, 0.0f }), 3, 3, + 10f, null, null, null, @@ -2666,6 +2672,7 @@ public void testByteVectorQueryBoundaries() throws IOException { VectorData.fromFloats(new float[] { 0, Float.NEGATIVE_INFINITY, 0.0f }), 3, 3, + 10f, null, null, null, @@ -2702,6 +2709,7 @@ public void testFloatVectorQueryBoundaries() throws IOException { VectorData.fromFloats(new float[] { Float.NaN, 0f, 0.0f }), 3, 3, + 10f, null, null, null, @@ -2718,6 +2726,7 @@ public void testFloatVectorQueryBoundaries() throws IOException { VectorData.fromFloats(new float[] { Float.POSITIVE_INFINITY, 0f, 0.0f }), 3, 3, + 10f, null, null, null, @@ -2737,6 +2746,7 @@ public void testFloatVectorQueryBoundaries() throws IOException { VectorData.fromFloats(new float[] { 0, Float.NEGATIVE_INFINITY, 0.0f }), 3, 3, + 10f, null, null, null, diff --git a/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java index 6ab936bfab27c..39520db299f65 100644 --- a/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java @@ -270,6 +270,7 @@ public void testKnnRewriteForInnerHits() throws IOException { new float[] { 1.0f, 2.0f, 3.0f }, null, 1, + 10f, null, null ); diff --git a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java index a8d9b1259cb41..b48d73aaa43ba 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java @@ -89,6 +89,7 @@ abstract KnnVectorQueryBuilder createKnnVectorQueryBuilder( String fieldName, int k, int numCands, + float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity ); @@ -145,10 +146,12 @@ protected KnnVectorQueryBuilder doCreateTestQueryBuilder() { String fieldName = randomBoolean() ? VECTOR_FIELD : VECTOR_ALIAS_FIELD; int k = randomIntBetween(1, 100); int numCands = randomIntBetween(k + 20, 1000); + float visitPercentage = randomFloatBetween(0.0f, 100.0f, true); KnnVectorQueryBuilder queryBuilder = createKnnVectorQueryBuilder( fieldName, k, numCands, + visitPercentage, isIndextypeBBQ() ? randomBBQRescoreVectorBuilder() : randomRescoreVectorBuilder(), randomFloat() ); @@ -284,7 +287,7 @@ protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query que public void testWrongDimension() { SearchExecutionContext context = createSearchExecutionContext(); - KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f }, 5, 10, null, null); + KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f }, 5, 10, 10f,null, null); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context)); assertThat( e.getMessage(), @@ -294,7 +297,7 @@ public void testWrongDimension() { public void testNonexistentField() { SearchExecutionContext context = createSearchExecutionContext(); - KnnVectorQueryBuilder query = new KnnVectorQueryBuilder("nonexistent", new float[] { 1.0f, 1.0f, 1.0f }, 5, 10, null, null); + KnnVectorQueryBuilder query = new KnnVectorQueryBuilder("nonexistent", new float[] { 1.0f, 1.0f, 1.0f }, 5, 10, 10f, null, null); context.setAllowUnmappedFields(false); QueryShardException e = expectThrows(QueryShardException.class, () -> query.doToQuery(context)); assertThat(e.getMessage(), containsString("No field mapping can be found for the field with name [nonexistent]")); @@ -302,7 +305,7 @@ public void testNonexistentField() { public void testNonexistentFieldReturnEmpty() throws IOException { SearchExecutionContext context = createSearchExecutionContext(); - KnnVectorQueryBuilder query = new KnnVectorQueryBuilder("nonexistent", new float[] { 1.0f, 1.0f, 1.0f }, 5, 10, null, null); + KnnVectorQueryBuilder query = new KnnVectorQueryBuilder("nonexistent", new float[] { 1.0f, 1.0f, 1.0f }, 5, 10, 10f, null, null); Query queryNone = query.doToQuery(context); assertThat(queryNone, instanceOf(MatchNoDocsQuery.class)); } @@ -314,6 +317,7 @@ public void testWrongFieldType() { new float[] { 1.0f, 1.0f, 1.0f }, 5, 10, + 10f, null, null ); @@ -326,14 +330,14 @@ public void testNumCandsLessThanK() { int numCands = 3; IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 1.0f, 1.0f }, k, numCands, null, null) + () -> new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 1.0f, 1.0f }, k, numCands, 10f, null, null) ); assertThat(e.getMessage(), containsString("[num_candidates] cannot be less than [k]")); } @Override public void testValidOutput() { - KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, null, 10, null, null); + KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, null, 10, 10f, null, null); String expected = """ { "knn" : { @@ -348,7 +352,7 @@ public void testValidOutput() { }"""; assertEquals(expected, query.toString()); - KnnVectorQueryBuilder query2 = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, 5, 10, null, null); + KnnVectorQueryBuilder query2 = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, 5, 10, 10f, null, null); String expected2 = """ { "knn" : { @@ -376,6 +380,7 @@ public void testMustRewrite() throws IOException { vectorDimensions, null, null, + null, null ); query.addFilterQuery(termQuery); @@ -395,6 +400,7 @@ public void testBWCVersionSerializationFilters() throws IOException { vectorData, null, query.numCands(), + query.visitPercentage(), null, null ).queryName(query.queryName()).boost(query.boost()); @@ -414,6 +420,7 @@ public void testBWCVersionSerializationSimilarity() throws IOException { vectorData, null, query.numCands(), + query.visitPercentage(), null, null ).queryName(query.queryName()).boost(query.boost()).addFilterQueries(query.filterQueries()); @@ -434,6 +441,7 @@ public void testBWCVersionSerializationQuery() throws IOException { vectorData, null, query.numCands(), + query.visitPercentage(), null, similarity ).queryName(query.queryName()).boost(query.boost()).addFilterQueries(query.filterQueries()); @@ -456,12 +464,37 @@ public void testBWCVersionSerializationRescoreVector() throws IOException { vectorData, k, query.numCands(), + query.visitPercentage(), null, query.getVectorSimilarity() ).queryName(query.queryName()).boost(query.boost()).addFilterQueries(query.filterQueries()); assertBWCSerialization(query, queryNoRescoreVector, version); } + // FIXME: write me + public void testBWCVersionSerializationVisitPercentage() throws IOException { + KnnVectorQueryBuilder query = createTestQueryBuilder(); + TransportVersion version = TransportVersionUtils.randomVersionBetween( + random(), + TransportVersions.V_9_0_0, + TransportVersions.V_9_1_0 + ); + VectorData vectorData = version.onOrAfter(TransportVersions.V_9_0_0) + ? query.queryVector() + : VectorData.fromFloats(query.queryVector().asFloatVector()); + Float visitPercentage = version.before(TransportVersions.V_9_1_0) ? null : query.visitPercentage(); + KnnVectorQueryBuilder queryVisitPercentage = new KnnVectorQueryBuilder( + query.getFieldName(), + vectorData, + query.k(), + query.numCands(), + visitPercentage, + null, + query.getVectorSimilarity() + ).queryName(query.queryName()).boost(query.boost()).addFilterQueries(query.filterQueries()); + assertBWCSerialization(query, queryVisitPercentage, version); + } + private void assertBWCSerialization(QueryBuilder newQuery, QueryBuilder bwcQuery, TransportVersion version) throws IOException { assertSerialization(bwcQuery, version); try (BytesStreamOutput output = new BytesStreamOutput()) { @@ -510,6 +543,7 @@ public void testRewriteWithQueryVectorBuilder() throws Exception { new TestQueryVectorBuilderPlugin.TestQueryVectorBuilder(expectedArray), null, 5, + 10f, 1f ); knnVectorQueryBuilder.boost(randomFloat()); diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java index 26066389c63f1..86bfde01c9faa 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java @@ -22,6 +22,7 @@ protected KnnVectorQueryBuilder createKnnVectorQueryBuilder( String fieldName, int k, int numCands, + float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity ) { @@ -29,7 +30,7 @@ protected KnnVectorQueryBuilder createKnnVectorQueryBuilder( for (int i = 0; i < vector.length; i++) { vector[i] = randomByte(); } - return new KnnVectorQueryBuilder(fieldName, vector, k, numCands, rescoreVectorBuilder, similarity); + return new KnnVectorQueryBuilder(fieldName, vector, k, numCands, visitPercentage, rescoreVectorBuilder, similarity); } @Override diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnFloatVectorQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnFloatVectorQueryBuilderTests.java index 70d29ab525ef1..13f9a8e441e0a 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnFloatVectorQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnFloatVectorQueryBuilderTests.java @@ -22,6 +22,7 @@ KnnVectorQueryBuilder createKnnVectorQueryBuilder( String fieldName, int k, int numCands, + float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity ) { @@ -29,7 +30,7 @@ KnnVectorQueryBuilder createKnnVectorQueryBuilder( for (int i = 0; i < vector.length; i++) { vector[i] = randomFloat(); } - return new KnnVectorQueryBuilder(fieldName, vector, k, numCands, rescoreVectorBuilder, similarity); + return new KnnVectorQueryBuilder(fieldName, vector, k, numCands, visitPercentage, rescoreVectorBuilder, similarity); } @Override diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java index 33ab8324ffb96..288b83deb2ef5 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java @@ -52,6 +52,7 @@ public static KnnSearchBuilder randomTestInstance() { float[] vector = randomVector(dim); int k = randomIntBetween(1, 100); int numCands = randomIntBetween(k + 20, 1000); + float visitPercentage = randomFloatBetween(0.0f, 100.0f, true); RescoreVectorBuilder rescoreVectorBuilder = randomBoolean() ? null : new RescoreVectorBuilder(randomFloatBetween(1.0f, 10.0f, false)); @@ -61,6 +62,7 @@ public static KnnSearchBuilder randomTestInstance() { vector, k, numCands, + visitPercentage, rescoreVectorBuilder, randomBoolean() ? null : randomFloat() ); @@ -110,7 +112,7 @@ protected KnnSearchBuilder createTestInstance() { @Override protected KnnSearchBuilder mutateInstance(KnnSearchBuilder instance) { - return switch (random().nextInt(8)) { + return switch (random().nextInt(9)) { case 0 -> { String newField = randomValueOtherThan(instance.field, () -> randomAlphaOfLength(5)); yield new KnnSearchBuilder( @@ -118,6 +120,7 @@ yield new KnnSearchBuilder( instance.queryVector, instance.k, instance.numCands, + instance.visitPercentage, instance.getRescoreVectorBuilder(), instance.similarity ).boost(instance.boost); @@ -129,6 +132,7 @@ yield new KnnSearchBuilder( newVector, instance.k, instance.numCands, + instance.visitPercentage, instance.getRescoreVectorBuilder(), instance.similarity ).boost(instance.boost); @@ -141,6 +145,7 @@ yield new KnnSearchBuilder( instance.queryVector, newK, instance.numCands, + instance.visitPercentage, instance.getRescoreVectorBuilder(), instance.similarity ).boost(instance.boost); @@ -152,6 +157,7 @@ yield new KnnSearchBuilder( instance.queryVector, instance.k, newNumCands, + instance.visitPercentage, instance.getRescoreVectorBuilder(), instance.similarity ).boost(instance.boost); @@ -161,6 +167,7 @@ yield new KnnSearchBuilder( instance.queryVector, instance.k, instance.numCands, + instance.visitPercentage, instance.getRescoreVectorBuilder(), instance.similarity ).addFilterQueries(instance.filterQueries) @@ -173,6 +180,7 @@ yield new KnnSearchBuilder( instance.queryVector, instance.k, instance.numCands, + instance.visitPercentage, instance.getRescoreVectorBuilder(), instance.similarity ).addFilterQueries(instance.filterQueries).boost(newBoost); @@ -182,6 +190,7 @@ yield new KnnSearchBuilder( instance.queryVector, instance.k, instance.numCands, + instance.visitPercentage, instance.getRescoreVectorBuilder(), randomValueOtherThan(instance.similarity, ESTestCase::randomFloat) ).addFilterQueries(instance.filterQueries).boost(instance.boost); @@ -190,12 +199,26 @@ yield new KnnSearchBuilder( instance.queryVector, instance.k, instance.numCands, + instance.visitPercentage, randomValueOtherThan( instance.getRescoreVectorBuilder(), () -> new RescoreVectorBuilder(randomFloatBetween(1.0f, 10.0f, false)) ), instance.similarity ).addFilterQueries(instance.filterQueries).boost(instance.boost); + case 8 -> { + Float newVisitPercentage = randomValueOtherThan(instance.visitPercentage, () -> instance.visitPercentage + + ESTestCase.randomFloatBetween(-100f, 100f, true)); + yield new KnnSearchBuilder( + instance.field, + instance.queryVector, + instance.k, + instance.numCands, + newVisitPercentage, + instance.getRescoreVectorBuilder(), + instance.similarity + ).boost(instance.boost); + } default -> throw new IllegalStateException(); }; } @@ -205,11 +228,12 @@ public void testToQueryBuilder() { float[] vector = randomVector(randomIntBetween(2, 30)); int k = randomIntBetween(1, 100); int numCands = randomIntBetween(k, 1000); + float visitPercentage = randomFloatBetween(0.0f, 100.0f, true); Float similarity = randomBoolean() ? null : randomFloat(); RescoreVectorBuilder rescoreVectorBuilder = randomBoolean() ? null : new RescoreVectorBuilder(randomFloatBetween(1.0f, 10.0f, false)); - KnnSearchBuilder builder = new KnnSearchBuilder(field, vector, k, numCands, rescoreVectorBuilder, similarity); + KnnSearchBuilder builder = new KnnSearchBuilder(field, vector, k, numCands, visitPercentage, rescoreVectorBuilder, similarity); float boost = AbstractQueryBuilder.DEFAULT_BOOST; if (randomBoolean()) { @@ -225,7 +249,8 @@ public void testToQueryBuilder() { builder.addFilterQuery(filter); } - QueryBuilder expected = new KnnVectorQueryBuilder(field, vector, numCands, numCands, rescoreVectorBuilder, similarity) + QueryBuilder expected = new KnnVectorQueryBuilder(field, vector, numCands, numCands, + visitPercentage, rescoreVectorBuilder, similarity) .addFilterQueries(filterQueries) .boost(boost); assertEquals(expected, builder.toQueryBuilder()); @@ -234,7 +259,7 @@ public void testToQueryBuilder() { public void testNumCandsLessThanK() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> new KnnSearchBuilder("field", randomVector(3), 50, 10, null, null) + () -> new KnnSearchBuilder("field", randomVector(3), 50, 10, 10f, null, null) ); assertThat(e.getMessage(), containsString("[num_candidates] cannot be less than [k]")); } @@ -242,15 +267,31 @@ public void testNumCandsLessThanK() { public void testNumCandsExceedsLimit() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> new KnnSearchBuilder("field", randomVector(3), 100, 10002, null, null) + () -> new KnnSearchBuilder("field", randomVector(3), 100, 10002, 10f, null, null) ); assertThat(e.getMessage(), containsString("[num_candidates] cannot exceed [10000]")); } + public void testVisitPercentageLessThan0() { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new KnnSearchBuilder("field", randomVector(3), 50, 10, -190f, null, null) + ); + assertThat(e.getMessage(), containsString("[visit_percentage] cannot be less than [0.0]")); + } + + public void testVisitPercentageGreaterThan100() { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new KnnSearchBuilder("field", randomVector(3), 100, 10002, 100000f, null, null) + ); + assertThat(e.getMessage(), containsString("[visit_percentage] cannot be greater than [100.0]")); + } + public void testInvalidK() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> new KnnSearchBuilder("field", randomVector(3), 0, 100, null, null) + () -> new KnnSearchBuilder("field", randomVector(3), 0, 100, 10f, null, null) ); assertThat(e.getMessage(), containsString("[k] must be greater than 0")); } @@ -258,7 +299,7 @@ public void testInvalidK() { public void testInvalidRescoreVectorBuilder() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> new KnnSearchBuilder("field", randomVector(3), 10, 100, new RescoreVectorBuilder(0.99F), null) + () -> new KnnSearchBuilder("field", randomVector(3), 10, 100, 10f, new RescoreVectorBuilder(0.99F), null) ); assertThat(e.getMessage(), containsString("[oversample] must be >= 1.0")); } @@ -271,6 +312,7 @@ public void testRewrite() throws Exception { new TestQueryVectorBuilderPlugin.TestQueryVectorBuilder(expectedArray), 5, 10, + 10f, expectedRescore, 1f ); diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java index 4e4d2158a9574..4e0006acb8e76 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java @@ -111,6 +111,7 @@ public void testParseSourceString() throws IOException { .field(KnnSearch.FIELD_FIELD.getPreferredName(), knnSearch.field) .field(KnnSearch.K_FIELD.getPreferredName(), knnSearch.k) .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), knnSearch.numCands) + .field(KnnSearch.VISIT_PERCENTAGE_FIELD.getPreferredName(), knnSearch.visitPercentage) .field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), knnSearch.queryVector) .endObject(); @@ -137,6 +138,7 @@ public void testParseSourceArray() throws IOException { .field(KnnSearch.FIELD_FIELD.getPreferredName(), knnSearch.field) .field(KnnSearch.K_FIELD.getPreferredName(), knnSearch.k) .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), knnSearch.numCands) + .field(KnnSearch.VISIT_PERCENTAGE_FIELD.getPreferredName(), knnSearch.visitPercentage) .field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), knnSearch.queryVector) .endObject(); @@ -195,6 +197,40 @@ public void testNumCandsExceedsLimit() throws IOException { assertThat(e.getMessage(), containsString("[num_candidates] cannot exceed [10000]")); } + public void testVisitPercnetageLessThan0() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()) + .startObject() + .startObject(KnnSearchRequestParser.KNN_SECTION_FIELD.getPreferredName()) + .field(KnnSearch.FIELD_FIELD.getPreferredName(), "field") + .field(KnnSearch.K_FIELD.getPreferredName(), 100) + .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), 1000) + .field(KnnSearch.VISIT_PERCENTAGE_FIELD.getPreferredName(), -100f) + .field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), new float[] { 1.0f, 2.0f, 3.0f }) + .endObject() + .endObject(); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseSearchRequest(builder)); + assertThat(e.getMessage(), containsString("[visit_percentage] cannot be less than [0.0f]")); + } + + public void testVisitPercnetageGreaterThan100() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()) + .startObject() + .startObject(KnnSearchRequestParser.KNN_SECTION_FIELD.getPreferredName()) + .field(KnnSearch.FIELD_FIELD.getPreferredName(), "field") + .field(KnnSearch.K_FIELD.getPreferredName(), 100) + .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), 1000) + .field(KnnSearch.VISIT_PERCENTAGE_FIELD.getPreferredName(), 1000f) + .field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), new float[] { 1.0f, 2.0f, 3.0f }) + .endObject() + .endObject(); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseSearchRequest(builder)); + assertThat(e.getMessage(), containsString("[visit_percentage] cannot be greater than [100.0f]")); + } + public void testInvalidK() throws IOException { XContentType xContentType = randomFrom(XContentType.values()); XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()) @@ -238,7 +274,8 @@ private KnnSearch randomKnnSearch() { int k = randomIntBetween(1, 100); int numCands = randomIntBetween(k, 1000); - return new KnnSearch(field, vector, k, numCands); + float visitPercentage = randomFloatBetween(0.0f, 100.0f, true); + return new KnnSearch(field, vector, k, numCands, visitPercentage); } private List randomFilterQueries() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java index b218b897121df..bc2a406f46219 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java @@ -23,6 +23,7 @@ import static org.elasticsearch.search.vectors.KnnVectorQueryBuilder.K_FIELD; import static org.elasticsearch.search.vectors.KnnVectorQueryBuilder.NUM_CANDS_FIELD; import static org.elasticsearch.search.vectors.KnnVectorQueryBuilder.VECTOR_SIMILARITY_FIELD; +import static org.elasticsearch.search.vectors.KnnVectorQueryBuilder.VISIT_PERCENTAGE_FIELD; public class KnnQuery extends Query { @@ -46,6 +47,7 @@ public KnnQuery(Source source, String field, float[] query, Map protected QueryBuilder asBuilder() { Integer k = (Integer) options.get(K_FIELD.getPreferredName()); Integer numCands = (Integer) options.get(NUM_CANDS_FIELD.getPreferredName()); + Float visitPercentage = (Float) options.get(VISIT_PERCENTAGE_FIELD.getPreferredName()); RescoreVectorBuilder rescoreVectorBuilder = null; Float oversample = (Float) options.get(RESCORE_OVERSAMPLE_FIELD); if (oversample != null) { @@ -53,7 +55,8 @@ protected QueryBuilder asBuilder() { } Float vectorSimilarity = (Float) options.get(VECTOR_SIMILARITY_FIELD.getPreferredName()); - KnnVectorQueryBuilder queryBuilder = new KnnVectorQueryBuilder(field, query, k, numCands, rescoreVectorBuilder, vectorSimilarity); + KnnVectorQueryBuilder queryBuilder = new KnnVectorQueryBuilder(field, query, k, numCands, + visitPercentage, rescoreVectorBuilder, vectorSimilarity); for (QueryBuilder filter : filterQueries) { queryBuilder.addFilterQuery(filter); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 9b161388d6cc3..d3b5216fc176e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -1394,6 +1394,7 @@ public void testKnnOptionsPushDown() { new float[] { 0.1f, 0.2f, 0.3f }, 5, 10, + 10f, new RescoreVectorBuilder(7), 0.001f ).boost(3.5f); @@ -1867,6 +1868,7 @@ public void testKnnPrefilters() { 10, null, null, + null, null ).addFilterQuery(expectedFilterQueryBuilder); var expectedQuery = boolQuery().must(expectedKnnQueryBuilder).must(expectedFilterQueryBuilder); @@ -1903,6 +1905,7 @@ public void testKnnPrefiltersWithMultipleFilters() { 10, null, null, + null, null ).addFilterQuery(expectedFilterQueryBuilder); var expectedQuery = boolQuery().must(expectedKnnQueryBuilder).must(integerFilter).must(keywordFilter); @@ -1938,6 +1941,7 @@ public void testPushDownConjunctionsToKnnPrefilter() { 10, null, null, + null, null ).addFilterQuery(expectedFilterQueryBuilder); @@ -1975,6 +1979,7 @@ public void testPushDownNegatedConjunctionsToKnnPrefilter() { 10, null, null, + null, null ).addFilterQuery(expectedFilterQueryBuilder); @@ -1999,7 +2004,8 @@ public void testNotPushDownDisjunctionsToKnnPrefilter() { var queryExec = as(field.child(), EsQueryExec.class); // The disjunction should not be pushed down to the KNN query - KnnVectorQueryBuilder knnQueryBuilder = new KnnVectorQueryBuilder("dense_vector", new float[] { 0, 1, 2 }, 10, null, null, null); + KnnVectorQueryBuilder knnQueryBuilder = new KnnVectorQueryBuilder("dense_vector", new float[] { 0, 1, 2 }, 10, + null, null,null, null); QueryBuilder rangeQueryBuilder = wrapWithSingleQuery( query, unscore(rangeQuery("integer").gt(10)), @@ -2082,8 +2088,8 @@ and NOT ((keyword == "test") or knn(dense_vector, [4, 5, 6], 10))) new Source(2, 46, "NOT integer > 10") ); - KnnVectorQueryBuilder firstKnn = new KnnVectorQueryBuilder("dense_vector", new float[] { 0, 1, 2 }, 10, null, null, null); - KnnVectorQueryBuilder secondKnn = new KnnVectorQueryBuilder("dense_vector", new float[] { 4, 5, 6 }, 10, null, null, null); + KnnVectorQueryBuilder firstKnn = new KnnVectorQueryBuilder("dense_vector", new float[] { 0, 1, 2 }, 10, null, null, null,null); + KnnVectorQueryBuilder secondKnn = new KnnVectorQueryBuilder("dense_vector", new float[] { 4, 5, 6 }, 10, null, null, null,null); firstKnn.addFilterQuery(notKeywordFilter); secondKnn.addFilterQuery(notIntegerGt10); @@ -2111,7 +2117,7 @@ public void testMultipleKnnQueriesInPrefilters() { var field = as(project.child(), FieldExtractExec.class); var queryExec = as(field.child(), EsQueryExec.class); - KnnVectorQueryBuilder firstKnnQuery = new KnnVectorQueryBuilder("dense_vector", new float[] { 0, 1, 2 }, 10, null, null, null); + KnnVectorQueryBuilder firstKnnQuery = new KnnVectorQueryBuilder("dense_vector", new float[] { 0, 1, 2 }, 10, null, null, null,null); // Integer range query (right side of first OR) QueryBuilder integerRangeQuery = wrapWithSingleQuery( query, @@ -2121,7 +2127,8 @@ public void testMultipleKnnQueriesInPrefilters() { ); // Second KNN query (right side of second OR) - KnnVectorQueryBuilder secondKnnQuery = new KnnVectorQueryBuilder("dense_vector", new float[] { 4, 5, 6 }, 10, null, null, null); + KnnVectorQueryBuilder secondKnnQuery = new KnnVectorQueryBuilder("dense_vector", new float[] { 4, 5, 6 }, 10, + null, null, null,null); // Keyword term query (left side of second OR) QueryBuilder keywordQuery = wrapWithSingleQuery( @@ -2661,7 +2668,7 @@ private static Object randomVector() { @Override public QueryBuilder queryBuilder() { - return new KnnVectorQueryBuilder(fieldName(), (float[]) queryString(), k, null, null, null); + return new KnnVectorQueryBuilder(fieldName(), (float[]) queryString(), k, null, null,null, null); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 50ca0e61eaeb8..5b35c0384ce99 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -970,7 +970,7 @@ yield new SparseVectorQueryBuilder( k = Math.max(k, DEFAULT_SIZE); } - yield new KnnVectorQueryBuilder(inferenceResultsFieldName, inference, k, null, null, null); + yield new KnnVectorQueryBuilder(inferenceResultsFieldName, inference, k, null, null, null, null); } default -> throw new IllegalStateException( "Field [" diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticKnnVectorQueryRewriteInterceptor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticKnnVectorQueryRewriteInterceptor.java index b1f5c240371f8..afa2b0182a0a4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticKnnVectorQueryRewriteInterceptor.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticKnnVectorQueryRewriteInterceptor.java @@ -141,6 +141,7 @@ private KnnVectorQueryBuilder addIndexFilterToKnnVectorQuery(Collection original.queryVectorBuilder(), original.k(), original.numCands(), + original.visitPercentage(), original.getVectorSimilarity() ); } else { @@ -149,6 +150,7 @@ private KnnVectorQueryBuilder addIndexFilterToKnnVectorQuery(Collection original.queryVector(), original.k(), original.numCands(), + original.visitPercentage(), original.rescoreVectorBuilder(), original.getVectorSimilarity() ); @@ -180,6 +182,7 @@ private KnnVectorQueryBuilder buildNewKnnVectorQuery( queryVectorBuilder, original.k(), original.numCands(), + original.visitPercentage(), original.getVectorSimilarity() ); } else { @@ -188,6 +191,7 @@ private KnnVectorQueryBuilder buildNewKnnVectorQuery( original.queryVector(), original.k(), original.numCands(), + original.visitPercentage(), original.rescoreVectorBuilder(), original.getVectorSimilarity() ); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/index/query/SemanticKnnVectorQueryRewriteInterceptorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/index/query/SemanticKnnVectorQueryRewriteInterceptorTests.java index 1f0b56e3d6848..751f5280682bc 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/index/query/SemanticKnnVectorQueryRewriteInterceptorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/index/query/SemanticKnnVectorQueryRewriteInterceptorTests.java @@ -60,7 +60,7 @@ public void testKnnQueryWithVectorBuilderIsInterceptedAndRewritten() throws IOEx ); QueryRewriteContext context = createQueryRewriteContext(inferenceFields); QueryVectorBuilder queryVectorBuilder = new TextEmbeddingQueryVectorBuilder(INFERENCE_ID, QUERY); - KnnVectorQueryBuilder original = new KnnVectorQueryBuilder(FIELD_NAME, queryVectorBuilder, 10, 100, null); + KnnVectorQueryBuilder original = new KnnVectorQueryBuilder(FIELD_NAME, queryVectorBuilder, 10, 100, 10f, null); if (randomBoolean()) { float boost = randomFloatBetween(1, 10, randomBoolean()); original.boost(boost); @@ -79,7 +79,7 @@ public void testKnnWithQueryBuilderWithoutInferenceIdIsInterceptedAndRewritten() ); QueryRewriteContext context = createQueryRewriteContext(inferenceFields); QueryVectorBuilder queryVectorBuilder = new TextEmbeddingQueryVectorBuilder(null, QUERY); - KnnVectorQueryBuilder original = new KnnVectorQueryBuilder(FIELD_NAME, queryVectorBuilder, 10, 100, null); + KnnVectorQueryBuilder original = new KnnVectorQueryBuilder(FIELD_NAME, queryVectorBuilder, 10, 100, 10f, null); if (randomBoolean()) { float boost = randomFloatBetween(1, 10, randomBoolean()); original.boost(boost); @@ -124,7 +124,7 @@ private void testRewrittenInferenceQuery(QueryRewriteContext context, KnnVectorQ public void testKnnVectorQueryOnNonInferenceFieldRemainsUnchanged() throws IOException { QueryRewriteContext context = createQueryRewriteContext(Map.of()); // No inference fields QueryVectorBuilder queryVectorBuilder = new TextEmbeddingQueryVectorBuilder(null, QUERY); - QueryBuilder original = new KnnVectorQueryBuilder(FIELD_NAME, queryVectorBuilder, 10, 100, null); + QueryBuilder original = new KnnVectorQueryBuilder(FIELD_NAME, queryVectorBuilder, 10, 100, 10f, null); QueryBuilder rewritten = original.rewrite(context); assertTrue( "Expected query to remain knn but was [" + rewritten.getClass().getName() + "]", diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java index 67c6d8da52a88..9d6488c13667e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java @@ -99,7 +99,8 @@ public void testDenseVector() throws Exception { Map queryMap = (Map) queries.get("dense_vector_1"); float[] vector = readDenseVector(queryMap.get("embeddings")); var fieldType = (SemanticTextFieldMapper.SemanticTextFieldType) mapperService.mappingLookup().getFieldType(SEMANTIC_FIELD_E5); - KnnVectorQueryBuilder knnQuery = new KnnVectorQueryBuilder(fieldType.getEmbeddingsField().fullPath(), vector, 10, 10, null, null); + KnnVectorQueryBuilder knnQuery = new KnnVectorQueryBuilder(fieldType.getEmbeddingsField().fullPath(), vector, 10, 10, + 10f, null, null); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder(fieldType.getChunksField().fullPath(), knnQuery, ScoreMode.Max); var shardRequest = createShardSearchRequest(nestedQueryBuilder); var sourceToParse = new SourceToParse("0", readSampleDoc(useLegacyFormat), XContentType.JSON); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverTelemetryTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverTelemetryTests.java index 7f6bc6117561b..82c11dcf8955d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverTelemetryTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverTelemetryTests.java @@ -116,7 +116,7 @@ public void testTelemetryForRRFRetriever() throws IOException { { performSearch( new SearchSourceBuilder().retriever( - new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", new float[] { 1.0f }, 10, 15, null, null)) + new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", new float[] { 1.0f }, 10, 15, 10f, null, null)) ) ); } @@ -149,7 +149,7 @@ public void testTelemetryForRRFRetriever() throws IOException { // search#6 - this will record 1 entry for "knn" in `sections` { performSearch( - new SearchSourceBuilder().knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 1.0f }, 10, 15, null, null))) + new SearchSourceBuilder().knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 1.0f }, 10, 15, 10f, null, null))) ); } diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverIT.java index b00af1713dcb6..3d01530d6125f 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverIT.java @@ -817,7 +817,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } }; var knn = new KnnRetrieverBuilder("vector", null, vectorBuilder, 10, 10, null, null); - var standard = new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", vectorBuilder, 10, 10, null)); + var standard = new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", vectorBuilder, 10, 10, 10f, null)); var rrf = new LinearRetrieverBuilder( List.of(new CompoundRetrieverBuilder.RetrieverSource(knn, null), new CompoundRetrieverBuilder.RetrieverSource(standard, null)), 10 diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java index 6854fc436038f..ad5a6464f12cd 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java @@ -819,7 +819,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } }; var knn = new KnnRetrieverBuilder("vector", null, vectorBuilder, 10, 10, null, null); - var standard = new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", vectorBuilder, 10, 10, null)); + var standard = new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", vectorBuilder, 10, 10, 10f, null)); var rrf = new RRFRetrieverBuilder( List.of(new CompoundRetrieverBuilder.RetrieverSource(knn, null), new CompoundRetrieverBuilder.RetrieverSource(standard, null)), 10, diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverTelemetryIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverTelemetryIT.java index 556f8b87923db..9944bd0111d9d 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverTelemetryIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverTelemetryIT.java @@ -119,7 +119,7 @@ public void testTelemetryForRRFRetriever() throws IOException { { performSearch( new SearchSourceBuilder().retriever( - new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", new float[] { 1.0f }, 10, 15, null, null)) + new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", new float[] { 1.0f }, 10, 15, 10f, null, null)) ) ); } @@ -156,7 +156,7 @@ public void testTelemetryForRRFRetriever() throws IOException { // search#6 - this will record 1 entry for "knn" in `sections` { performSearch( - new SearchSourceBuilder().knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 1.0f }, 10, 15, null, null))) + new SearchSourceBuilder().knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 1.0f }, 10, 15, 10f, null, null))) ); } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java index 2c379ce7352f0..af48bd43a704a 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java @@ -892,7 +892,7 @@ public void testKnnSearch() throws Exception { // Since there's no kNN search action at the transport layer, we just emulate // how the action works (it builds a kNN query under the hood) float[] queryVector = new float[] { 0.0f, 0.0f, 0.0f }; - KnnVectorQueryBuilder query = new KnnVectorQueryBuilder("vector", queryVector, 50, 50, null, null); + KnnVectorQueryBuilder query = new KnnVectorQueryBuilder("vector", queryVector, 50, 50, 10f, null, null); if (randomBoolean()) { query.addFilterQuery(new WildcardQueryBuilder("other", "value*")); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java index fadabb4e8fcb3..f067dea6245fe 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java @@ -441,7 +441,7 @@ public void testKnnSearch() throws IOException { // Since there's no kNN search action at the transport layer, we just emulate // how the action works (it builds a kNN query under the hood) float[] queryVector = new float[] { 0.0f, 0.0f, 0.0f }; - KnnVectorQueryBuilder query = new KnnVectorQueryBuilder("vector", queryVector, 10, 10, null, null); + KnnVectorQueryBuilder query = new KnnVectorQueryBuilder("vector", queryVector, 10, 10, 10f, null, null); // user1 has access to vector field, so the query should match with the document: assertResponse( @@ -475,7 +475,7 @@ public void testKnnSearch() throws IOException { } ); // user1 can access field1, so the filtered query should match with the document: - KnnVectorQueryBuilder filterQuery1 = new KnnVectorQueryBuilder("vector", queryVector, 10, 10, null, null).addFilterQuery( + KnnVectorQueryBuilder filterQuery1 = new KnnVectorQueryBuilder("vector", queryVector, 10, 10, 10f, null, null).addFilterQuery( QueryBuilders.matchQuery("field1", "value1") ); assertHitCount( @@ -486,7 +486,7 @@ public void testKnnSearch() throws IOException { ); // user1 cannot access field2, so the filtered query should not match with the document: - KnnVectorQueryBuilder filterQuery2 = new KnnVectorQueryBuilder("vector", queryVector, 10, 10, null, null).addFilterQuery( + KnnVectorQueryBuilder filterQuery2 = new KnnVectorQueryBuilder("vector", queryVector, 10, 10, 10f, null, null).addFilterQuery( QueryBuilders.matchQuery("field2", "value2") ); assertHitCount( diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SemanticTextUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SemanticTextUpgradeIT.java index f21525103d37d..9dcf5537ee9d5 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SemanticTextUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SemanticTextUpgradeIT.java @@ -194,7 +194,7 @@ private ObjectPath semanticQuery(String field, Model fieldModel, String query, I Arrays.fill(queryVector, 1.0f); } - yield new KnnVectorQueryBuilder(embeddingsFieldName, queryVector, DOC_VALUES.size(), null, null, null); + yield new KnnVectorQueryBuilder(embeddingsFieldName, queryVector, DOC_VALUES.size(), null, null, null, null); } default -> throw new UnsupportedOperationException("Unhandled task type [" + fieldModel.getTaskType() + "]"); }; From 2ab8989bb95366f925025407a27f26cf0c8b4073 Mon Sep 17 00:00:00 2001 From: John Wagster Date: Thu, 28 Aug 2025 14:27:36 -0500 Subject: [PATCH 02/36] spotless --- .../percolator/PercolatorQuerySearchIT.java | 2 +- .../DenseVectorFieldIndexTypeUpdateIT.java | 2 +- .../retriever/RetrieverTelemetryIT.java | 2 +- .../search/vectors/KnnSearchBuilder.java | 19 ++++----- .../vectors/KnnSearchRequestParser.java | 2 +- .../search/vectors/KnnVectorQueryBuilder.java | 42 ++++++++++++++----- ...AbstractKnnVectorQueryBuilderTestCase.java | 2 +- .../search/vectors/KnnSearchBuilderTests.java | 19 ++++++--- .../xpack/esql/querydsl/query/KnnQuery.java | 11 ++++- .../LocalPhysicalPlanOptimizerTests.java | 38 +++++++++++++---- .../SemanticTextHighlighterTests.java | 11 ++++- 11 files changed, 106 insertions(+), 44 deletions(-) diff --git a/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java index 3aaf668851ded..5e8ced116a1ff 100644 --- a/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java @@ -1359,7 +1359,7 @@ public void testKnnQueryNotSupportedInPercolator() throws IOException { """); indicesAdmin().prepareCreate("index1").setMapping(mappings).get(); ensureGreen(); - QueryBuilder knnVectorQueryBuilder = new KnnVectorQueryBuilder("my_vector", new float[] { 1, 1, 1, 1, 1 }, 10, 10, 10f,null, null); + QueryBuilder knnVectorQueryBuilder = new KnnVectorQueryBuilder("my_vector", new float[] { 1, 1, 1, 1, 1 }, 10, 10, 10f, null, null); IndexRequestBuilder indexRequestBuilder = prepareIndex("index1").setId("knn_query1") .setSource(jsonBuilder().startObject().field("my_query", knnVectorQueryBuilder).endObject()); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldIndexTypeUpdateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldIndexTypeUpdateIT.java index 6066d53605c99..4d698107dd0e6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldIndexTypeUpdateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldIndexTypeUpdateIT.java @@ -142,7 +142,7 @@ public void testDenseVectorMappingUpdate() throws Exception { for (int i = 0; i < queryVector.length; i++) { queryVector[i] = randomFloatBetween(-1, 1, true); } - KnnVectorQueryBuilder queryBuilder = new KnnVectorQueryBuilder(VECTOR_FIELD, queryVector, null, null, null,null, null); + KnnVectorQueryBuilder queryBuilder = new KnnVectorQueryBuilder(VECTOR_FIELD, queryVector, null, null, null, null, null); assertNoFailuresAndResponse( client().prepareSearch(INDEX_NAME).setQuery(queryBuilder).setTrackTotalHits(true).setSize(expectedDocs), response -> { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverTelemetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverTelemetryIT.java index 8256d19f12d84..56f1e51113bc9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverTelemetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverTelemetryIT.java @@ -99,7 +99,7 @@ public void testTelemetryForRetrievers() throws IOException { { performSearch( new SearchSourceBuilder().retriever( - new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", new float[] { 1.0f }, 10, 15, 10f,null, null)) + new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", new float[] { 1.0f }, 10, 15, 10f, null, null)) ) ); } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index cf83e1df20f68..9a57e7a425aee 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -436,11 +436,9 @@ public KnnSearchBuilder rewrite(QueryRewriteContext ctx) throws IOException { if (querySupplier.get() == null) { return this; } - return new KnnSearchBuilder(field, querySupplier.get(), k, numCands, - visitPercentage, rescoreVectorBuilder, similarity).boost(boost) - .queryName(queryName) - .addFilterQueries(filterQueries) - .innerHit(innerHitBuilder); + return new KnnSearchBuilder(field, querySupplier.get(), k, numCands, visitPercentage, rescoreVectorBuilder, similarity).boost( + boost + ).queryName(queryName).addFilterQueries(filterQueries).innerHit(innerHitBuilder); } if (queryVectorBuilder != null) { SetOnce toSet = new SetOnce<>(); @@ -460,8 +458,8 @@ public KnnSearchBuilder rewrite(QueryRewriteContext ctx) throws IOException { } ll.onResponse(null); }))); - return new KnnSearchBuilder(field, toSet::get, k, numCands, - visitPercentage, rescoreVectorBuilder, filterQueries, similarity).boost(boost) + return new KnnSearchBuilder(field, toSet::get, k, numCands, visitPercentage, rescoreVectorBuilder, filterQueries, similarity) + .boost(boost) .queryName(queryName) .innerHit(innerHitBuilder); } @@ -487,10 +485,9 @@ public KnnVectorQueryBuilder toQueryBuilder() { if (queryVectorBuilder != null) { throw new IllegalArgumentException("missing rewrite"); } - return new KnnVectorQueryBuilder(field, queryVector, numCands, numCands, - visitPercentage, rescoreVectorBuilder, similarity).boost(boost) - .queryName(queryName) - .addFilterQueries(filterQueries); + return new KnnVectorQueryBuilder(field, queryVector, numCands, numCands, visitPercentage, rescoreVectorBuilder, similarity).boost( + boost + ).queryName(queryName).addFilterQueries(filterQueries); } public Float getSimilarity() { diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java index 5ab90c570f57a..12cb674f1c9a2 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java @@ -259,7 +259,7 @@ public KnnVectorQueryBuilder toQueryBuilder() { if (numCands > NUM_CANDS_LIMIT) { throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot exceed [" + NUM_CANDS_LIMIT + "]"); } - if(visitPercentage < 0.0f || visitPercentage > 100.0f) { + if (visitPercentage < 0.0f || visitPercentage > 100.0f) { throw new IllegalArgumentException("[" + VISIT_PERCENTAGE_FIELD.getPreferredName() + "] must be between 0 and 100"); } return new KnnVectorQueryBuilder(field, queryVector, numCands, numCands, visitPercentage, null, null); diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index 77abdefd98d4a..5a4cfbdb0a165 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -141,8 +141,17 @@ public KnnVectorQueryBuilder( RescoreVectorBuilder rescoreVectorBuilder, Float vectorSimilarity ) { - this(fieldName, VectorData.fromFloats(queryVector), null, null, k, numCands, - visitPercentage, rescoreVectorBuilder, vectorSimilarity); + this( + fieldName, + VectorData.fromFloats(queryVector), + null, + null, + k, + numCands, + visitPercentage, + rescoreVectorBuilder, + vectorSimilarity + ); } public KnnVectorQueryBuilder( @@ -165,8 +174,17 @@ public KnnVectorQueryBuilder( RescoreVectorBuilder rescoreVectorBuilder, Float vectorSimilarity ) { - this(fieldName, VectorData.fromBytes(queryVector), null, null, k, numCands, - visitPercentage, rescoreVectorBuilder, vectorSimilarity); + this( + fieldName, + VectorData.fromBytes(queryVector), + null, + null, + k, + numCands, + visitPercentage, + rescoreVectorBuilder, + vectorSimilarity + ); } public KnnVectorQueryBuilder( @@ -245,7 +263,7 @@ public KnnVectorQueryBuilder(StreamInput in) throws IOException { this.numCands = in.readVInt(); } // FIXME: validate transport version changes - if(in.getTransportVersion().onOrAfter(TransportVersions.V_9_1_0)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_9_1_0)) { this.visitPercentage = in.readOptionalFloat(); } else { this.visitPercentage = in.readFloat(); @@ -443,11 +461,15 @@ protected QueryBuilder doRewrite(QueryRewriteContext ctx) throws IOException { if (queryVectorSupplier.get() == null) { return this; } - return new KnnVectorQueryBuilder(fieldName, queryVectorSupplier.get(), k, numCands, - visitPercentage, rescoreVectorBuilder, vectorSimilarity) - .boost(boost) - .queryName(queryName) - .addFilterQueries(filterQueries); + return new KnnVectorQueryBuilder( + fieldName, + queryVectorSupplier.get(), + k, + numCands, + visitPercentage, + rescoreVectorBuilder, + vectorSimilarity + ).boost(boost).queryName(queryName).addFilterQueries(filterQueries); } if (queryVectorBuilder != null) { SetOnce toSet = new SetOnce<>(); diff --git a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java index b48d73aaa43ba..34d73ad92eb64 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java @@ -287,7 +287,7 @@ protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query que public void testWrongDimension() { SearchExecutionContext context = createSearchExecutionContext(); - KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f }, 5, 10, 10f,null, null); + KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f }, 5, 10, 10f, null, null); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context)); assertThat( e.getMessage(), diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java index 288b83deb2ef5..eb4663d8226c9 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java @@ -207,8 +207,10 @@ yield new KnnSearchBuilder( instance.similarity ).addFilterQueries(instance.filterQueries).boost(instance.boost); case 8 -> { - Float newVisitPercentage = randomValueOtherThan(instance.visitPercentage, () -> instance.visitPercentage + - ESTestCase.randomFloatBetween(-100f, 100f, true)); + Float newVisitPercentage = randomValueOtherThan( + instance.visitPercentage, + () -> instance.visitPercentage + ESTestCase.randomFloatBetween(-100f, 100f, true) + ); yield new KnnSearchBuilder( instance.field, instance.queryVector, @@ -249,10 +251,15 @@ public void testToQueryBuilder() { builder.addFilterQuery(filter); } - QueryBuilder expected = new KnnVectorQueryBuilder(field, vector, numCands, numCands, - visitPercentage, rescoreVectorBuilder, similarity) - .addFilterQueries(filterQueries) - .boost(boost); + QueryBuilder expected = new KnnVectorQueryBuilder( + field, + vector, + numCands, + numCands, + visitPercentage, + rescoreVectorBuilder, + similarity + ).addFilterQueries(filterQueries).boost(boost); assertEquals(expected, builder.toQueryBuilder()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java index bc2a406f46219..feec90a808e81 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java @@ -55,8 +55,15 @@ protected QueryBuilder asBuilder() { } Float vectorSimilarity = (Float) options.get(VECTOR_SIMILARITY_FIELD.getPreferredName()); - KnnVectorQueryBuilder queryBuilder = new KnnVectorQueryBuilder(field, query, k, numCands, - visitPercentage, rescoreVectorBuilder, vectorSimilarity); + KnnVectorQueryBuilder queryBuilder = new KnnVectorQueryBuilder( + field, + query, + k, + numCands, + visitPercentage, + rescoreVectorBuilder, + vectorSimilarity + ); for (QueryBuilder filter : filterQueries) { queryBuilder.addFilterQuery(filter); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index d3b5216fc176e..feba5628791f9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -2004,8 +2004,15 @@ public void testNotPushDownDisjunctionsToKnnPrefilter() { var queryExec = as(field.child(), EsQueryExec.class); // The disjunction should not be pushed down to the KNN query - KnnVectorQueryBuilder knnQueryBuilder = new KnnVectorQueryBuilder("dense_vector", new float[] { 0, 1, 2 }, 10, - null, null,null, null); + KnnVectorQueryBuilder knnQueryBuilder = new KnnVectorQueryBuilder( + "dense_vector", + new float[] { 0, 1, 2 }, + 10, + null, + null, + null, + null + ); QueryBuilder rangeQueryBuilder = wrapWithSingleQuery( query, unscore(rangeQuery("integer").gt(10)), @@ -2088,8 +2095,8 @@ and NOT ((keyword == "test") or knn(dense_vector, [4, 5, 6], 10))) new Source(2, 46, "NOT integer > 10") ); - KnnVectorQueryBuilder firstKnn = new KnnVectorQueryBuilder("dense_vector", new float[] { 0, 1, 2 }, 10, null, null, null,null); - KnnVectorQueryBuilder secondKnn = new KnnVectorQueryBuilder("dense_vector", new float[] { 4, 5, 6 }, 10, null, null, null,null); + KnnVectorQueryBuilder firstKnn = new KnnVectorQueryBuilder("dense_vector", new float[] { 0, 1, 2 }, 10, null, null, null, null); + KnnVectorQueryBuilder secondKnn = new KnnVectorQueryBuilder("dense_vector", new float[] { 4, 5, 6 }, 10, null, null, null, null); firstKnn.addFilterQuery(notKeywordFilter); secondKnn.addFilterQuery(notIntegerGt10); @@ -2117,7 +2124,15 @@ public void testMultipleKnnQueriesInPrefilters() { var field = as(project.child(), FieldExtractExec.class); var queryExec = as(field.child(), EsQueryExec.class); - KnnVectorQueryBuilder firstKnnQuery = new KnnVectorQueryBuilder("dense_vector", new float[] { 0, 1, 2 }, 10, null, null, null,null); + KnnVectorQueryBuilder firstKnnQuery = new KnnVectorQueryBuilder( + "dense_vector", + new float[] { 0, 1, 2 }, + 10, + null, + null, + null, + null + ); // Integer range query (right side of first OR) QueryBuilder integerRangeQuery = wrapWithSingleQuery( query, @@ -2127,8 +2142,15 @@ public void testMultipleKnnQueriesInPrefilters() { ); // Second KNN query (right side of second OR) - KnnVectorQueryBuilder secondKnnQuery = new KnnVectorQueryBuilder("dense_vector", new float[] { 4, 5, 6 }, 10, - null, null, null,null); + KnnVectorQueryBuilder secondKnnQuery = new KnnVectorQueryBuilder( + "dense_vector", + new float[] { 4, 5, 6 }, + 10, + null, + null, + null, + null + ); // Keyword term query (left side of second OR) QueryBuilder keywordQuery = wrapWithSingleQuery( @@ -2668,7 +2690,7 @@ private static Object randomVector() { @Override public QueryBuilder queryBuilder() { - return new KnnVectorQueryBuilder(fieldName(), (float[]) queryString(), k, null, null,null, null); + return new KnnVectorQueryBuilder(fieldName(), (float[]) queryString(), k, null, null, null, null); } @Override diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java index 9d6488c13667e..8284907a1873c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java @@ -99,8 +99,15 @@ public void testDenseVector() throws Exception { Map queryMap = (Map) queries.get("dense_vector_1"); float[] vector = readDenseVector(queryMap.get("embeddings")); var fieldType = (SemanticTextFieldMapper.SemanticTextFieldType) mapperService.mappingLookup().getFieldType(SEMANTIC_FIELD_E5); - KnnVectorQueryBuilder knnQuery = new KnnVectorQueryBuilder(fieldType.getEmbeddingsField().fullPath(), vector, 10, 10, - 10f, null, null); + KnnVectorQueryBuilder knnQuery = new KnnVectorQueryBuilder( + fieldType.getEmbeddingsField().fullPath(), + vector, + 10, + 10, + 10f, + null, + null + ); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder(fieldType.getChunksField().fullPath(), knnQuery, ScoreMode.Max); var shardRequest = createShardSearchRequest(nestedQueryBuilder); var sourceToParse = new SourceToParse("0", readSampleDoc(useLegacyFormat), XContentType.JSON); From e947aed3ffb272f861408f010ed1768532e83cd5 Mon Sep 17 00:00:00 2001 From: John Wagster Date: Thu, 28 Aug 2025 16:50:48 -0500 Subject: [PATCH 03/36] iter --- .../elasticsearch/index/store/DirectIOIT.java | 2 +- .../search/nested/VectorNestedIT.java | 5 ++- .../search/profile/dfs/DfsProfilerIT.java | 1 + .../elasticsearch/search/query/VectorIT.java | 4 +- .../org/elasticsearch/TransportVersions.java | 2 +- .../search/retriever/KnnRetrieverBuilder.java | 14 ++++++- .../search/vectors/KnnSearchBuilder.java | 12 +++++- .../vectors/KnnSearchRequestParser.java | 4 ++ .../search/vectors/KnnVectorQueryBuilder.java | 24 ++++++++++- .../action/search/DfsQueryPhaseTests.java | 4 +- .../action/search/SearchRequestTests.java | 17 ++++---- .../action/search/RestSearchActionTests.java | 2 +- .../builder/SearchSourceBuilderTests.java | 2 +- .../search/dfs/DfsPhaseTests.java | 2 +- ...AbstractKnnVectorQueryBuilderTestCase.java | 4 +- .../vectors/KnnSearchRequestParserTests.java | 1 + .../search/RandomSearchRequestGenerator.java | 3 +- .../AbstractQueryVectorBuilderTestCase.java | 3 ++ .../esql/expression/function/vector/Knn.java | 2 + .../xpack/rank/rrf/RRFRankMultiShardIT.java | 40 +++++++++---------- .../xpack/rank/rrf/RRFRankSingleShardIT.java | 34 ++++++++-------- .../rank/rrf/RRFRetrieverTelemetryIT.java | 4 +- .../xpack/rank/rrf/RRFRankBuilder.java | 1 + 23 files changed, 122 insertions(+), 65 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/DirectIOIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/DirectIOIT.java index b14f067992ba0..e27ba0e141491 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/DirectIOIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/DirectIOIT.java @@ -124,7 +124,7 @@ public void testDirectIOUsed() { indexVectors(); // do a search - var knn = List.of(new KnnSearchBuilder("fooVector", new VectorData(null, new byte[64]), 10, 20, null, null)); + var knn = List.of(new KnnSearchBuilder("fooVector", new VectorData(null, new byte[64]), 10, 20, 10f, null, null)); assertHitCount(prepareSearch("foo-vectors").setKnnSearch(knn), 10); mockLog.assertAllExpectationsMatched(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/VectorNestedIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/VectorNestedIT.java index cbe7e7be51902..4824d89f713f7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/VectorNestedIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/VectorNestedIT.java @@ -72,7 +72,8 @@ public void testSimpleNested() throws Exception { assertResponse( prepareSearch("test").setKnnSearch( - List.of(new KnnSearchBuilder("nested.vector", new float[] { 1, 1, 1 }, 1, 1, null, null).innerHit(new InnerHitBuilder())) + List.of(new KnnSearchBuilder("nested.vector", new float[] { 1, 1, 1 }, 1, 1, + 10f, null, null).innerHit(new InnerHitBuilder())) ).setAllowPartialSearchResults(false), response -> assertThat(response.getHits().getHits().length, greaterThan(0)) ); @@ -153,7 +154,7 @@ private void testNestedWithTwoSegments(boolean flush) { waitForRelocation(ClusterHealthStatus.GREEN); refresh(); - var knn = new KnnSearchBuilder("nested.vector", new float[] { -0.5f, 90.0f, -10f, 14.8f, -156.0f }, 2, 3, null, null); + var knn = new KnnSearchBuilder("nested.vector", new float[] { -0.5f, 90.0f, -10f, 14.8f, -156.0f }, 2, 3, 10f, null, null); var request = prepareSearch("test").addFetchField("name").setKnnSearch(List.of(knn)); assertNoFailuresAndResponse(request, response -> { assertHitCount(response, 2); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java index 95d69a6ebaa86..c84e955ec8ce5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java @@ -72,6 +72,7 @@ public void testProfileDfs() throws Exception { new float[] { randomFloat(), randomFloat(), randomFloat() }, randomIntBetween(5, 10), 50, + 10f, randomBoolean() ? null : new RescoreVectorBuilder(randomFloatBetween(1.0f, 10.0f, false)), randomBoolean() ? null : randomFloat() ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/VectorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/VectorIT.java index 3dabe1b37b43e..e2c1fd0b35b69 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/VectorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/VectorIT.java @@ -75,7 +75,7 @@ public void testFilteredQueryStrategy() { float[] vector = new float[16]; randomVector(vector, 25); int upperLimit = 35; - var query = new KnnSearchBuilder(VECTOR_FIELD, vector, 1, 1, null, null).addFilterQuery( + var query = new KnnSearchBuilder(VECTOR_FIELD, vector, 1, 1, 10f, null, null).addFilterQuery( QueryBuilders.rangeQuery(NUM_ID_FIELD).lte(35) ); assertResponse(client().prepareSearch(INDEX_NAME).setKnnSearch(List.of(query)).setSize(1).setProfile(true), acornResponse -> { @@ -131,7 +131,7 @@ public void testHnswEarlyTerminationQuery() { float[] vector = new float[16]; randomVector(vector, 25); int upperLimit = 35; - var query = new KnnSearchBuilder(VECTOR_FIELD, vector, 1, 1, null, null); + var query = new KnnSearchBuilder(VECTOR_FIELD, vector, 1, 1, 10f, null, null); assertResponse(client().prepareSearch(INDEX_NAME).setKnnSearch(List.of(query)).setSize(1).setProfile(true), response -> { assertNotEquals(0, response.getHits().getHits().length); var profileResults = response.getProfileResults(); diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index f82440f620d89..292339caae913 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -224,7 +224,6 @@ static TransportVersion def(int id) { public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_4 = def(9_000_0_13); public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_6 = def(9_000_0_15); public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_7 = def(9_000_0_16); - public static final TransportVersion V_9_1_0 = def(9_001_0_00); public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_001_0_00); public static final TransportVersion REMOVE_SNAPSHOT_FAILURES = def(9_002_0_00); public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED = def(9_003_0_00); @@ -358,6 +357,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_SAMPLE_OPERATOR_STATUS = def(9_127_0_00); public static final TransportVersion ALLOCATION_DECISION_NOT_PREFERRED = def(9_145_0_00); public static final TransportVersion ESQL_QUALIFIERS_IN_ATTRIBUTES = def(9_146_0_00); + public static final TransportVersion VISIT_PERCENTAGE = def(9_147_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java index 6db6b29515d21..8657bfa3b6afd 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java @@ -48,6 +48,7 @@ public final class KnnRetrieverBuilder extends RetrieverBuilder { public static final ParseField FIELD_FIELD = new ParseField("field"); public static final ParseField K_FIELD = new ParseField("k"); public static final ParseField NUM_CANDS_FIELD = new ParseField("num_candidates"); + public static final ParseField VISIT_PERCENTAGE_FIELD = new ParseField("visit_percentage"); public static final ParseField QUERY_VECTOR_FIELD = new ParseField("query_vector"); public static final ParseField QUERY_VECTOR_BUILDER_FIELD = new ParseField("query_vector_builder"); public static final ParseField VECTOR_SIMILARITY = new ParseField("similarity"); @@ -73,8 +74,9 @@ public final class KnnRetrieverBuilder extends RetrieverBuilder { (QueryVectorBuilder) args[2], (int) args[3], (int) args[4], - (RescoreVectorBuilder) args[6], - (Float) args[5] + (float) args[5], + (RescoreVectorBuilder) args[7], + (Float) args[6] ); } ); @@ -89,6 +91,7 @@ public final class KnnRetrieverBuilder extends RetrieverBuilder { ); PARSER.declareInt(constructorArg(), K_FIELD); PARSER.declareInt(constructorArg(), NUM_CANDS_FIELD); + PARSER.declareFloat(constructorArg(), VISIT_PERCENTAGE_FIELD); PARSER.declareFloat(optionalConstructorArg(), VECTOR_SIMILARITY); PARSER.declareField( optionalConstructorArg(), @@ -108,6 +111,7 @@ public static KnnRetrieverBuilder fromXContent(XContentParser parser, RetrieverP private final QueryVectorBuilder queryVectorBuilder; private final int k; private final int numCands; + private final float visitPercentage; private final RescoreVectorBuilder rescoreVectorBuilder; private final Float similarity; @@ -117,6 +121,7 @@ public KnnRetrieverBuilder( QueryVectorBuilder queryVectorBuilder, int k, int numCands, + float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity ) { @@ -142,6 +147,7 @@ public KnnRetrieverBuilder( this.queryVectorBuilder = queryVectorBuilder; this.k = k; this.numCands = numCands; + this.visitPercentage = visitPercentage; this.similarity = similarity; this.rescoreVectorBuilder = rescoreVectorBuilder; } @@ -152,6 +158,7 @@ private KnnRetrieverBuilder(KnnRetrieverBuilder clone, Supplier queryVe this.field = clone.field; this.k = clone.k; this.numCands = clone.numCands; + this.visitPercentage = clone.visitPercentage; this.similarity = clone.similarity; this.retrieverName = clone.retrieverName; this.preFilterQueryBuilders = clone.preFilterQueryBuilders; @@ -236,6 +243,7 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder null, k, numCands, + visitPercentage, rescoreVectorBuilder, similarity ); @@ -261,6 +269,7 @@ public void doToXContent(XContentBuilder builder, Params params) throws IOExcept builder.field(FIELD_FIELD.getPreferredName(), field); builder.field(K_FIELD.getPreferredName(), k); builder.field(NUM_CANDS_FIELD.getPreferredName(), numCands); + builder.field(VISIT_PERCENTAGE_FIELD.getPreferredName(), visitPercentage); if (queryVector != null) { builder.field(QUERY_VECTOR_FIELD.getPreferredName(), queryVector.get()); @@ -284,6 +293,7 @@ public boolean doEquals(Object o) { KnnRetrieverBuilder that = (KnnRetrieverBuilder) o; return k == that.k && numCands == that.numCands + && visitPercentage == that.visitPercentage && Objects.equals(field, that.field) && ((queryVector == null && that.queryVector == null) || (queryVector != null && that.queryVector != null && Arrays.equals(queryVector.get(), that.queryVector.get()))) diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index 9a57e7a425aee..37b143e9fd3c6 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -49,6 +49,7 @@ public class KnnSearchBuilder implements Writeable, ToXContentFragment, Rewritea public static final ParseField FIELD_FIELD = new ParseField("field"); public static final ParseField K_FIELD = new ParseField("k"); public static final ParseField NUM_CANDS_FIELD = new ParseField("num_candidates"); + public static final ParseField VISIT_PERCENTAGE = new ParseField("visit_percentage"); public static final ParseField QUERY_VECTOR_FIELD = new ParseField("query_vector"); public static final ParseField QUERY_VECTOR_BUILDER_FIELD = new ParseField("query_vector_builder"); public static final ParseField VECTOR_SIMILARITY = new ParseField("similarity"); @@ -80,6 +81,7 @@ public class KnnSearchBuilder implements Writeable, ToXContentFragment, Rewritea ); PARSER.declareInt(optionalConstructorArg(), K_FIELD); PARSER.declareInt(optionalConstructorArg(), NUM_CANDS_FIELD); + PARSER.declareFloat(optionalConstructorArg(), VISIT_PERCENTAGE); PARSER.declareNamedObject( optionalConstructorArg(), (p, c, n) -> p.namedObject(QueryVectorBuilder.class, n, c), @@ -279,7 +281,7 @@ private KnnSearchBuilder( throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot exceed [" + NUM_CANDS_LIMIT + "]"); } if (visitPercentage < 0f || visitPercentage > 100f) { - throw new IllegalArgumentException("[" + "visit_percentage" + "] must be between [0] and [100]"); + throw new IllegalArgumentException("[" + VISIT_PERCENTAGE.getPreferredName() + "] must be between [0] and [100]"); } if (queryVector == null && queryVectorBuilder == null) { throw new IllegalArgumentException( @@ -318,7 +320,7 @@ public KnnSearchBuilder(StreamInput in) throws IOException { this.field = in.readString(); this.k = in.readVInt(); this.numCands = in.readVInt(); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_9_1_0)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.VISIT_PERCENTAGE)) { this.visitPercentage = in.readFloat(); } else { this.visitPercentage = 0f; @@ -364,6 +366,10 @@ public int getNumCands() { return numCands; } + public float getVisitPercentage() { + return visitPercentage; + } + public RescoreVectorBuilder getRescoreVectorBuilder() { return rescoreVectorBuilder; } @@ -501,6 +507,7 @@ public boolean equals(Object o) { KnnSearchBuilder that = (KnnSearchBuilder) o; return k == that.k && numCands == that.numCands + && visitPercentage == that.visitPercentage && Objects.equals(rescoreVectorBuilder, that.rescoreVectorBuilder) && Objects.equals(field, that.field) && Objects.equals(queryVector, that.queryVector) @@ -536,6 +543,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(FIELD_FIELD.getPreferredName(), field); builder.field(K_FIELD.getPreferredName(), k); builder.field(NUM_CANDS_FIELD.getPreferredName(), numCands); + builder.field(VISIT_PERCENTAGE.getPreferredName(), visitPercentage); if (queryVectorBuilder != null) { builder.startObject(QUERY_VECTOR_BUILDER_FIELD.getPreferredName()); diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java index 12cb674f1c9a2..0287285545b1f 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java @@ -199,6 +199,8 @@ static class KnnSearch { static final ParseField FIELD_FIELD = new ParseField("field"); static final ParseField K_FIELD = new ParseField("k"); static final ParseField NUM_CANDS_FIELD = new ParseField("num_candidates"); + // FIXME: update docs to reflect the new optional field + // FIXME: add more tests around this field static final ParseField VISIT_PERCENTAGE_FIELD = new ParseField("visit_percentage"); static final ParseField QUERY_VECTOR_FIELD = new ParseField("query_vector"); @@ -217,6 +219,7 @@ static class KnnSearch { PARSER.declareFloatArray(constructorArg(), QUERY_VECTOR_FIELD); PARSER.declareInt(constructorArg(), K_FIELD); PARSER.declareInt(constructorArg(), NUM_CANDS_FIELD); + PARSER.declareFloat(constructorArg(), VISIT_PERCENTAGE_FIELD); } public static KnnSearch parse(XContentParser parser) throws IOException { @@ -272,6 +275,7 @@ public boolean equals(Object o) { KnnSearch that = (KnnSearch) o; return k == that.k && numCands == that.numCands + && visitPercentage == that.visitPercentage && Objects.equals(field, that.field) && Arrays.equals(queryVector, that.queryVector); } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index 5a4cfbdb0a165..0405f9be71bd0 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -96,6 +96,7 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder 100.0f)) { + throw new IllegalArgumentException("[" + VISIT_PERCENTAGE_FIELD.getPreferredName() + "] must be between 0.0 and 100.0"); + } if (queryVector == null && queryVectorBuilder == null) { throw new IllegalArgumentException( format( @@ -263,7 +267,7 @@ public KnnVectorQueryBuilder(StreamInput in) throws IOException { this.numCands = in.readVInt(); } // FIXME: validate transport version changes - if (in.getTransportVersion().onOrAfter(TransportVersions.V_9_1_0)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.VISIT_PERCENTAGE)) { this.visitPercentage = in.readOptionalFloat(); } else { this.visitPercentage = in.readFloat(); @@ -377,6 +381,21 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeVInt(numCands); } } + if (out.getTransportVersion().onOrAfter(TransportVersions.VISIT_PERCENTAGE)) { + out.writeOptionalFloat(visitPercentage); + } else { + if (visitPercentage == null) { + throw new IllegalArgumentException( + "[" + + VISIT_PERCENTAGE_FIELD.getPreferredName() + + "] field was mandatory in previous releases " + + "and is required to be non-null by some nodes. " + + "Please make sure to provide the parameter as part of the request." + ); + } else { + out.writeFloat(visitPercentage); + } + } if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalWriteable(queryVector); } else { @@ -428,6 +447,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep if (numCands != null) { builder.field(NUM_CANDS_FIELD.getPreferredName(), numCands); } + if (visitPercentage != null) { + builder.field(VISIT_PERCENTAGE_FIELD.getPreferredName(), visitPercentage); + } if (vectorSimilarity != null) { builder.field(VECTOR_SIMILARITY_FIELD.getPreferredName(), vectorSimilarity); } diff --git a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java index a4f698d04b782..f468c0c346aa5 100644 --- a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java @@ -353,8 +353,8 @@ public void testRewriteShardSearchRequestWithRank() { SearchSourceBuilder ssb = new SearchSourceBuilder().query(bm25) .knnSearch( List.of( - new KnnSearchBuilder("vector", new float[] { 0.0f }, 10, 100, null, null), - new KnnSearchBuilder("vector2", new float[] { 0.0f }, 10, 100, null, null) + new KnnSearchBuilder("vector", new float[] { 0.0f }, 10, 100, 10f, null, null), + new KnnSearchBuilder("vector2", new float[] { 0.0f }, 10, 100, 10f, null, null) ) ) .rankBuilder(new TestRankBuilder(100)); diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java index 9d9132ecdffe8..25c4c1672e852 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java @@ -120,6 +120,7 @@ public void testSerializationMultiKNN() throws Exception { new float[] { 1, 2 }, 5, 10, + 10f, randomRescoreVectorBuilder(), randomBoolean() ? null : randomFloat() ), @@ -128,6 +129,7 @@ public void testSerializationMultiKNN() throws Exception { new float[] { 4, 12, 41 }, 3, 5, + 10f, randomRescoreVectorBuilder(), randomBoolean() ? null : randomFloat() ) @@ -151,6 +153,7 @@ public void testSerializationMultiKNN() throws Exception { new float[] { 1, 2 }, 5, 10, + 10f, randomRescoreVectorBuilder(), randomBoolean() ? null : randomFloat() ) @@ -474,7 +477,7 @@ public QueryBuilder topDocsQuery() { SearchRequest searchRequest = new SearchRequest().source( new SearchSourceBuilder().rankBuilder(new TestRankBuilder(100)) .query(QueryBuilders.termQuery("field", "term")) - .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, null, null))) + .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, 10f, null, null))) .size(0) ); ActionRequestValidationException validationErrors = searchRequest.validate(); @@ -486,7 +489,7 @@ public QueryBuilder topDocsQuery() { SearchRequest searchRequest = new SearchRequest().source( new SearchSourceBuilder().rankBuilder(new TestRankBuilder(1)) .query(QueryBuilders.termQuery("field", "term")) - .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, null, null))) + .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, 10f, null, null))) .size(2) ); ActionRequestValidationException validationErrors = searchRequest.validate(); @@ -513,7 +516,7 @@ public QueryBuilder topDocsQuery() { SearchRequest searchRequest = new SearchRequest().source( new SearchSourceBuilder().rankBuilder(new TestRankBuilder(100)) .query(QueryBuilders.termQuery("field", "term")) - .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, null, null))) + .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, 10f, null, null))) ).scroll(new TimeValue(1000)); ActionRequestValidationException validationErrors = searchRequest.validate(); assertNotNull(validationErrors); @@ -524,7 +527,7 @@ public QueryBuilder topDocsQuery() { SearchRequest searchRequest = new SearchRequest().source( new SearchSourceBuilder().rankBuilder(new TestRankBuilder(9)) .query(QueryBuilders.termQuery("field", "term")) - .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, null, null))) + .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, 10f, null, null))) ); ActionRequestValidationException validationErrors = searchRequest.validate(); assertNotNull(validationErrors); @@ -538,7 +541,7 @@ public QueryBuilder topDocsQuery() { SearchRequest searchRequest = new SearchRequest().source( new SearchSourceBuilder().rankBuilder(new TestRankBuilder(3)) .query(QueryBuilders.termQuery("field", "term")) - .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, null, null))) + .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, 10f, null, null))) .size(3) .from(4) ); @@ -549,7 +552,7 @@ public QueryBuilder topDocsQuery() { SearchRequest searchRequest = new SearchRequest().source( new SearchSourceBuilder().rankBuilder(new TestRankBuilder(100)) .query(QueryBuilders.termQuery("field", "term")) - .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, null, null))) + .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, 10f, null, null))) .addRescorer(new QueryRescorerBuilder(QueryBuilders.termQuery("rescore", "another term"))) ); ActionRequestValidationException validationErrors = searchRequest.validate(); @@ -561,7 +564,7 @@ public QueryBuilder topDocsQuery() { SearchRequest searchRequest = new SearchRequest().source( new SearchSourceBuilder().rankBuilder(new TestRankBuilder(100)) .query(QueryBuilders.termQuery("field", "term")) - .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, null, null))) + .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, 10f, null, null))) .suggest(new SuggestBuilder().setGlobalText("test").addSuggestion("suggestion", new TermSuggestionBuilder("term"))) ); ActionRequestValidationException validationErrors = searchRequest.validate(); diff --git a/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java index 580dad8128494..ef620896e941d 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java @@ -83,7 +83,7 @@ public void testValidateSearchRequest() { .build(); SearchRequest searchRequest = new SearchRequest(); - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", new float[] { 1, 1, 1 }, 10, 100, null, null); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", new float[] { 1, 1, 1 }, 10, 100, 10f, null, null); searchRequest.source(new SearchSourceBuilder().knnSearch(List.of(knnSearch))); Exception ex = expectThrows( diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index 83f19924ec3d6..dc6328862254a 100644 --- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -826,7 +826,7 @@ public void testSearchSectionsUsageCollection() throws IOException { searchSourceBuilder.fetchField("field"); // these are not correct runtime mappings but they are counted compared to empty object searchSourceBuilder.runtimeMappings(Collections.singletonMap("field", "keyword")); - searchSourceBuilder.knnSearch(List.of(new KnnSearchBuilder("field", new float[] {}, 2, 5, null, null))); + searchSourceBuilder.knnSearch(List.of(new KnnSearchBuilder("field", new float[] {}, 2, 5, 10f, null, null))); searchSourceBuilder.pointInTimeBuilder(new PointInTimeBuilder(new BytesArray("pitid"))); searchSourceBuilder.docValueField("field"); searchSourceBuilder.storedField("field"); diff --git a/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java b/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java index dd2bcd7175976..ea5e7e6b7a488 100644 --- a/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java @@ -156,7 +156,7 @@ public DfsSearchResult dfsResult() { context.request() .source( new SearchSourceBuilder().knnSearch( - List.of(new KnnSearchBuilder("float_vector", new float[] { 0, 0, 0 }, numDocs, numDocs, null, null)) + List.of(new KnnSearchBuilder("float_vector", new float[] { 0, 0, 0 }, numDocs, numDocs, 100f, null, null)) ) ); context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap())); diff --git a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java index 34d73ad92eb64..81f7d0f550c07 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java @@ -477,12 +477,12 @@ public void testBWCVersionSerializationVisitPercentage() throws IOException { TransportVersion version = TransportVersionUtils.randomVersionBetween( random(), TransportVersions.V_9_0_0, - TransportVersions.V_9_1_0 + TransportVersions.VISIT_PERCENTAGE ); VectorData vectorData = version.onOrAfter(TransportVersions.V_9_0_0) ? query.queryVector() : VectorData.fromFloats(query.queryVector().asFloatVector()); - Float visitPercentage = version.before(TransportVersions.V_9_1_0) ? null : query.visitPercentage(); + Float visitPercentage = version.before(TransportVersions.VISIT_PERCENTAGE) ? null : query.visitPercentage(); KnnVectorQueryBuilder queryVisitPercentage = new KnnVectorQueryBuilder( query.getFieldName(), vectorData, diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java index 4e0006acb8e76..00ca8de3de982 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java @@ -298,6 +298,7 @@ private XContentBuilder createRequestBody(KnnSearch knnSearch, List ALLOWED_OPTIONS = Map.ofEntries( entry(NUM_CANDS_FIELD.getPreferredName(), INTEGER), + entry(VISIT_PERCENTAGE_FIELD.getPreferredName(), FLOAT), entry(VECTOR_SIMILARITY_FIELD.getPreferredName(), FLOAT), entry(BOOST_FIELD.getPreferredName(), FLOAT), entry(KnnQuery.RESCORE_OVERSAMPLE_FIELD, FLOAT) diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankMultiShardIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankMultiShardIT.java index 457c57410d168..016507ac9c369 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankMultiShardIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankMultiShardIT.java @@ -136,7 +136,7 @@ public void setupSuiteScopeCluster() throws Exception { public void testTotalDocsSmallerThanSize() { float[] queryVector = { 0.0f }; - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 3, 3, null, null); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 3, 3, 10f, null, null); assertResponse( prepareSearch("tiny_index").setRankBuilder(new RRFRankBuilder(100, 1)) .setKnnSearch(List.of(knnSearch)) @@ -167,7 +167,7 @@ public void testTotalDocsSmallerThanSize() { public void testBM25AndKnn() { float[] queryVector = { 500.0f }; - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, null, null); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, 10f, null, null); assertResponse( prepareSearch("nrd_index").setRankBuilder(new RRFRankBuilder(101, 1)) .setTrackTotalHits(false) @@ -208,8 +208,8 @@ public void testBM25AndKnn() { public void testMultipleOnlyKnn() { float[] queryVectorAsc = { 500.0f }; float[] queryVectorDesc = { 500.0f }; - KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 51, 1001, null, null); - KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 51, 1001, null, null); + KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 51, 1001, 10f, null, null); + KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 51, 1001, 10f, null, null); assertResponse( prepareSearch("nrd_index").setRankBuilder(new RRFRankBuilder(51, 1)) .setTrackTotalHits(true) @@ -260,8 +260,8 @@ public void testMultipleOnlyKnn() { public void testBM25AndMultipleKnn() { float[] queryVectorAsc = { 500.0f }; float[] queryVectorDesc = { 500.0f }; - KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 51, 1001, null, null); - KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 51, 1001, null, null); + KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 51, 1001, 10f, null, null); + KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 51, 1001, 10f, null, null); assertResponse( prepareSearch("nrd_index").setRankBuilder(new RRFRankBuilder(51, 1)) .setTrackTotalHits(false) @@ -332,7 +332,7 @@ public void testBM25AndMultipleKnn() { public void testBM25AndKnnWithBucketAggregation() { float[] queryVector = { 500.0f }; - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, null, null); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, 10f, null, null); assertResponse( prepareSearch("nrd_index").setRankBuilder(new RRFRankBuilder(101, 1)) .setTrackTotalHits(true) @@ -389,8 +389,8 @@ public void testBM25AndKnnWithBucketAggregation() { public void testMultipleOnlyKnnWithAggregation() { float[] queryVectorAsc = { 500.0f }; float[] queryVectorDesc = { 500.0f }; - KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 51, 1001, null, null); - KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 51, 1001, null, null); + KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 51, 1001, 10f, null, null); + KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 51, 1001, 10f, null, null); assertResponse( prepareSearch("nrd_index").setRankBuilder(new RRFRankBuilder(51, 1)) .setTrackTotalHits(false) @@ -457,8 +457,8 @@ public void testMultipleOnlyKnnWithAggregation() { public void testBM25AndMultipleKnnWithAggregation() { float[] queryVectorAsc = { 500.0f }; float[] queryVectorDesc = { 500.0f }; - KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 51, 1001, null, null); - KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 51, 1001, null, null); + KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 51, 1001, 10f, null, null); + KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 51, 1001, 10f, null, null); assertResponse( prepareSearch("nrd_index").setRankBuilder(new RRFRankBuilder(51, 1)) .setTrackTotalHits(true) @@ -704,7 +704,7 @@ public void testMultiBM25WithAggregation() { public void testMultiBM25AndSingleKnn() { float[] queryVector = { 500.0f }; - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, null, null); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, 10f, null, null); assertResponse( prepareSearch("nrd_index").setRankBuilder(new RRFRankBuilder(101, 1)) .setTrackTotalHits(false) @@ -762,7 +762,7 @@ public void testMultiBM25AndSingleKnn() { public void testMultiBM25AndSingleKnnWithAggregation() { float[] queryVector = { 500.0f }; - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, null, null); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, 10f, null, null); assertResponse( prepareSearch("nrd_index").setRankBuilder(new RRFRankBuilder(101, 1)) .setTrackTotalHits(false) @@ -837,8 +837,8 @@ public void testMultiBM25AndSingleKnnWithAggregation() { public void testMultiBM25AndMultipleKnn() { float[] queryVectorAsc = { 500.0f }; float[] queryVectorDesc = { 500.0f }; - KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 101, 1001, null, null); - KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 101, 1001, null, null); + KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 101, 1001, 10f, null, null); + KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 101, 1001, 10f, null, null); assertResponse( prepareSearch("nrd_index").setRankBuilder(new RRFRankBuilder(101, 1)) .setTrackTotalHits(false) @@ -899,8 +899,8 @@ public void testMultiBM25AndMultipleKnn() { public void testMultiBM25AndMultipleKnnWithAggregation() { float[] queryVectorAsc = { 500.0f }; float[] queryVectorDesc = { 500.0f }; - KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 101, 1001, null, null); - KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 101, 1001, null, null); + KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 101, 1001, 10f, null, null); + KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 101, 1001, 10f, null, null); assertResponse( prepareSearch("nrd_index").setRankBuilder(new RRFRankBuilder(101, 1)) .setTrackTotalHits(false) @@ -979,7 +979,7 @@ public void testBasicRRFExplain() { // the first result should be the one present in both queries (i.e. doc with text0: 10 and vector: [10]) and the other ones // should only match the knn query float[] queryVector = { 9f }; - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, null, null).queryName("my_knn_search"); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, 10f, null, null).queryName("my_knn_search"); assertResponse( prepareSearch("nrd_index").setRankBuilder(new RRFRankBuilder(100, 1)) .setKnnSearch(List.of(knnSearch)) @@ -1045,7 +1045,7 @@ public void testRRFExplainUnknownField() { // in this test we try knn with a query on an unknown field that would be rewritten to MatchNoneQuery // so we expect results and explanations only for the first part float[] queryVector = { 9f }; - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, null, null).queryName("my_knn_search"); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, 10f, null, null).queryName("my_knn_search"); assertResponse( prepareSearch("nrd_index").setRankBuilder(new RRFRankBuilder(100, 1)) .setKnnSearch(List.of(knnSearch)) @@ -1112,7 +1112,7 @@ public void testRRFExplainOneUnknownFieldSubSearches() { // while the other one would produce a match. // So, we'd have a total of 3 queries, a (rewritten) MatchNoneQuery, a TermQuery, and a kNN query float[] queryVector = { 9f }; - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, null, null).queryName("my_knn_search"); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, 10f, null, null).queryName("my_knn_search"); assertResponse( prepareSearch("nrd_index").setRankBuilder(new RRFRankBuilder(100, 1)) .setKnnSearch(List.of(knnSearch)) diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankSingleShardIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankSingleShardIT.java index a4e7db3b3e3fe..364a241a643a4 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankSingleShardIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankSingleShardIT.java @@ -131,7 +131,7 @@ public void setupIndices() throws Exception { public void testTotalDocsSmallerThanSize() { float[] queryVector = { 0.0f }; - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 3, 3, null, null); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 3, 3, 10f, null, null); assertResponse( client().prepareSearch("tiny_index") @@ -164,7 +164,7 @@ public void testTotalDocsSmallerThanSize() { public void testBM25AndKnn() { float[] queryVector = { 500.0f }; - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, null, null); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, 10f, null, null); assertResponse( client().prepareSearch("nrd_index") .setRankBuilder(new RRFRankBuilder(101, 1)) @@ -206,8 +206,8 @@ public void testBM25AndKnn() { public void testMultipleOnlyKnn() { float[] queryVectorAsc = { 500.0f }; float[] queryVectorDesc = { 500.0f }; - KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 51, 1001, null, null); - KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 51, 1001, null, null); + KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 51, 1001, 10f, null, null); + KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 51, 1001, 10f, null, null); assertResponse( client().prepareSearch("nrd_index") .setRankBuilder(new RRFRankBuilder(51, 1)) @@ -259,8 +259,8 @@ public void testMultipleOnlyKnn() { public void testBM25AndMultipleKnn() { float[] queryVectorAsc = { 500.0f }; float[] queryVectorDesc = { 500.0f }; - KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 51, 1001, null, null); - KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 51, 1001, null, null); + KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 51, 1001, 10f, null, null); + KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 51, 1001, 10f, null, null); assertResponse( client().prepareSearch("nrd_index") .setRankBuilder(new RRFRankBuilder(51, 1)) @@ -332,7 +332,7 @@ public void testBM25AndMultipleKnn() { public void testBM25AndKnnWithBucketAggregation() { float[] queryVector = { 500.0f }; - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, null, null); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, 10f, null, null); assertResponse( client().prepareSearch("nrd_index") .setRankBuilder(new RRFRankBuilder(101, 1)) @@ -390,8 +390,8 @@ public void testBM25AndKnnWithBucketAggregation() { public void testMultipleOnlyKnnWithAggregation() { float[] queryVectorAsc = { 500.0f }; float[] queryVectorDesc = { 500.0f }; - KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 51, 1001, null, null); - KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 51, 1001, null, null); + KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 51, 1001, 10f, null, null); + KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 51, 1001, 10f, null, null); assertResponse( client().prepareSearch("nrd_index") .setRankBuilder(new RRFRankBuilder(51, 1)) @@ -459,8 +459,8 @@ public void testMultipleOnlyKnnWithAggregation() { public void testBM25AndMultipleKnnWithAggregation() { float[] queryVectorAsc = { 500.0f }; float[] queryVectorDesc = { 500.0f }; - KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 51, 1001, null, null); - KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 51, 1001, null, null); + KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 51, 1001, 10f, null, null); + KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 51, 1001, 10f, null, null); assertResponse( client().prepareSearch("nrd_index") .setRankBuilder(new RRFRankBuilder(51, 1)) @@ -709,7 +709,7 @@ public void testMultiBM25WithAggregation() { public void testMultiBM25AndSingleKnn() { float[] queryVector = { 500.0f }; - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, null, null); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, 10f, null, null); assertResponse( client().prepareSearch("nrd_index") .setRankBuilder(new RRFRankBuilder(101, 1)) @@ -768,7 +768,7 @@ public void testMultiBM25AndSingleKnn() { public void testMultiBM25AndSingleKnnWithAggregation() { float[] queryVector = { 500.0f }; - KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, null, null); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, 10f, null, null); assertResponse( client().prepareSearch("nrd_index") .setRankBuilder(new RRFRankBuilder(101, 1)) @@ -844,8 +844,8 @@ public void testMultiBM25AndSingleKnnWithAggregation() { public void testMultiBM25AndMultipleKnn() { float[] queryVectorAsc = { 500.0f }; float[] queryVectorDesc = { 500.0f }; - KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 101, 1001, null, null); - KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 101, 1001, null, null); + KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 101, 1001, 10f, null, null); + KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 101, 1001, 10f, null, null); assertResponse( client().prepareSearch("nrd_index") .setRankBuilder(new RRFRankBuilder(101, 1)) @@ -907,8 +907,8 @@ public void testMultiBM25AndMultipleKnn() { public void testMultiBM25AndMultipleKnnWithAggregation() { float[] queryVectorAsc = { 500.0f }; float[] queryVectorDesc = { 500.0f }; - KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 101, 1001, null, null); - KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 101, 1001, null, null); + KnnSearchBuilder knnSearchAsc = new KnnSearchBuilder("vector_asc", queryVectorAsc, 101, 1001, 10f, null, null); + KnnSearchBuilder knnSearchDesc = new KnnSearchBuilder("vector_desc", queryVectorDesc, 101, 1001, 10f, null, null); assertResponse( client().prepareSearch("nrd_index") .setRankBuilder(new RRFRankBuilder(101, 1)) diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverTelemetryIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverTelemetryIT.java index 9944bd0111d9d..2e18ddb8f3f76 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverTelemetryIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverTelemetryIT.java @@ -104,7 +104,7 @@ public void testTelemetryForRRFRetriever() throws IOException { // search#1 - this will record 1 entry for "retriever" in `sections`, and 1 for "knn" under `retrievers` { performSearch( - new SearchSourceBuilder().retriever(new KnnRetrieverBuilder("vector", new float[] { 1.0f }, null, 10, 15, null, null)) + new SearchSourceBuilder().retriever(new KnnRetrieverBuilder("vector", new float[] { 1.0f }, null, 10, 15, 10f,null, null)) ); } @@ -138,7 +138,7 @@ public void testTelemetryForRRFRetriever() throws IOException { new RRFRetrieverBuilder( Arrays.asList( new CompoundRetrieverBuilder.RetrieverSource( - new KnnRetrieverBuilder("vector", new float[] { 1.0f }, null, 10, 15, null, null), + new KnnRetrieverBuilder("vector", new float[] { 1.0f }, null, 10, 15, 10f, null, null), null ), new CompoundRetrieverBuilder.RetrieverSource( diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java index b6ffbf8f3301e..61ff39197a799 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java @@ -212,6 +212,7 @@ public RetrieverBuilder toRetriever(SearchSourceBuilder source, Predicate Date: Thu, 28 Aug 2025 22:01:07 +0000 Subject: [PATCH 04/36] [CI] Auto commit changes from spotless --- .../elasticsearch/search/nested/VectorNestedIT.java | 5 +++-- .../search/vectors/KnnVectorQueryBuilder.java | 2 +- .../search/RandomSearchRequestGenerator.java | 12 ++++++++++-- .../xpack/rank/rrf/RRFRetrieverTelemetryIT.java | 2 +- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/VectorNestedIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/VectorNestedIT.java index 4824d89f713f7..0ff2b7336e654 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/VectorNestedIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/VectorNestedIT.java @@ -72,8 +72,9 @@ public void testSimpleNested() throws Exception { assertResponse( prepareSearch("test").setKnnSearch( - List.of(new KnnSearchBuilder("nested.vector", new float[] { 1, 1, 1 }, 1, 1, - 10f, null, null).innerHit(new InnerHitBuilder())) + List.of( + new KnnSearchBuilder("nested.vector", new float[] { 1, 1, 1 }, 1, 1, 10f, null, null).innerHit(new InnerHitBuilder()) + ) ).setAllowPartialSearchResults(false), response -> assertThat(response.getHits().getHits().length, greaterThan(0)) ); diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index 0405f9be71bd0..90c769ca86b57 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -222,7 +222,7 @@ private KnnVectorQueryBuilder( "[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot be less than [" + K_FIELD.getPreferredName() + "]" ); } - if( visitPercentage != null && (visitPercentage < 0.0f || visitPercentage > 100.0f)) { + if (visitPercentage != null && (visitPercentage < 0.0f || visitPercentage > 100.0f)) { throw new IllegalArgumentException("[" + VISIT_PERCENTAGE_FIELD.getPreferredName() + "] must be between 0.0 and 100.0"); } if (queryVector == null && queryVectorBuilder == null) { diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index 73f34e76c4f20..5f91b76387211 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -266,12 +266,20 @@ public static SearchSourceBuilder randomSearchSourceBuilder( } int k = randomIntBetween(1, 100); int numCands = randomIntBetween(k, 1000); - float visitPercentage = randomFloatBetween(0.0f, 100.0f, true); + float visitPercentage = randomFloatBetween(0.0f, 100.0f, true); RescoreVectorBuilder rescoreVectorBuilder = randomBoolean() ? null : new RescoreVectorBuilder(randomFloatBetween(1.0f, 10.0f, false)); knnSearchBuilders.add( - new KnnSearchBuilder(field, vector, k, numCands, visitPercentage, rescoreVectorBuilder, randomBoolean() ? null : randomFloat()) + new KnnSearchBuilder( + field, + vector, + k, + numCands, + visitPercentage, + rescoreVectorBuilder, + randomBoolean() ? null : randomFloat() + ) ); } builder.knnSearch(knnSearchBuilders); diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverTelemetryIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverTelemetryIT.java index 2e18ddb8f3f76..2712ab84cf972 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverTelemetryIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverTelemetryIT.java @@ -104,7 +104,7 @@ public void testTelemetryForRRFRetriever() throws IOException { // search#1 - this will record 1 entry for "retriever" in `sections`, and 1 for "knn" under `retrievers` { performSearch( - new SearchSourceBuilder().retriever(new KnnRetrieverBuilder("vector", new float[] { 1.0f }, null, 10, 15, 10f,null, null)) + new SearchSourceBuilder().retriever(new KnnRetrieverBuilder("vector", new float[] { 1.0f }, null, 10, 15, 10f, null, null)) ); } From dc749b324142be05d36950f80f448544737fb0ad Mon Sep 17 00:00:00 2001 From: John Wagster Date: Thu, 28 Aug 2025 17:30:34 -0500 Subject: [PATCH 05/36] docs --- .../query-languages/query-dsl/query-dsl-knn-query.md | 8 ++++++-- .../xpack/esql/expression/function/vector/Knn.java | 10 ++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md b/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md index 4af581e466a7a..1b41d4cccfc51 100644 --- a/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md +++ b/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md @@ -87,6 +87,10 @@ If all queried fields are of type [semantic_text](/reference/elasticsearch/mappi : (Optional, integer) The number of nearest neighbor candidates to consider per shard while doing knn search. Cannot exceed 10,000. Increasing `num_candidates` tends to improve the accuracy of the final results. Defaults to `1.5 * k` if `k` is set, or `1.5 * size` if `k` is not set. +`visit_percentage` +: (Optional, float) The percentage of vectors to explore per shard while doing knn search with `bbq_disk`. Must be between 1 and 100. 0 will default to using `num_candidates` for calculating the percent visited. Increasing `visit_percentage` tends to improve the accuracy of the final results. If `visit_percentage` is set for `bbq_disk`, `num_candidates` is ignored. Defaults to a reasonable percent per shard of ~1% for every 1 million vectors. + + `filter` : (Optional, query object) Query to filter the documents that can match. The kNN search will return the top documents that also match this filter. The value can be a single query or a list of queries. If `filter` is not provided, all documents are allowed to match. @@ -108,7 +112,7 @@ The filter is a pre-filter, meaning that it is applied **during** the approximat : (Optional, object) Apply oversampling and rescoring to quantized vectors. **Parameters for `rescore_vector`**: - + `oversample` : (Required, float) @@ -116,7 +120,7 @@ The filter is a pre-filter, meaning that it is applied **during** the approximat * Retrieve `num_candidates` candidates per shard. * From these candidates, the top `k * oversample` candidates per shard will be rescored using the original vectors. - * The top `k` rescored candidates will be returned. Must be one of the following values: + * The top `k` rescored candidates will be returned. Must be one of the following values: * \>= 1f to indicate the oversample factor * Exactly `0` to indicate that no oversampling and rescoring should occur. {applies_to}`stack: ga 9.1` diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java index 3a0b62ab87f4a..d5f4706ba27f7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java @@ -132,6 +132,16 @@ public Knn( + "Cannot exceed 10,000. Increasing num_candidates tends to improve the accuracy of the final results. " + "Defaults to 1.5 * k" ), + @MapParam.MapParamEntry( + name = "visit_percentage", + type = "float", + valueHint = { "10.0" }, + description = "(Optional, float) The percentage of vectors to explore per shard while doing knn search " + + "with `bbq_disk`. Must be between 1 and 100. 0 will default to using `num_candidates` for calculating " + + "the percent visited. Increasing `visit_percentage` tends to improve the accuracy of the final results. " + + "If `visit_percentage` is set is set for `bbq_disk`, `num_candidates` is ignored. " + + "Defaults to a reasonable percent per shard of ~1% for every 1 million vectors." + ), @MapParam.MapParamEntry( name = "similarity", type = "double", From af20b61d20ce1bbe42014ce891ce0fa06ee97afd Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 28 Aug 2025 22:38:47 +0000 Subject: [PATCH 06/36] [CI] Auto commit changes from spotless --- .../xpack/esql/expression/function/vector/Knn.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java index d5f4706ba27f7..0141fcc5b881f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java @@ -136,11 +136,11 @@ public Knn( name = "visit_percentage", type = "float", valueHint = { "10.0" }, - description = "(Optional, float) The percentage of vectors to explore per shard while doing knn search " + - "with `bbq_disk`. Must be between 1 and 100. 0 will default to using `num_candidates` for calculating " + - "the percent visited. Increasing `visit_percentage` tends to improve the accuracy of the final results. " + - "If `visit_percentage` is set is set for `bbq_disk`, `num_candidates` is ignored. " + - "Defaults to a reasonable percent per shard of ~1% for every 1 million vectors." + description = "(Optional, float) The percentage of vectors to explore per shard while doing knn search " + + "with `bbq_disk`. Must be between 1 and 100. 0 will default to using `num_candidates` for calculating " + + "the percent visited. Increasing `visit_percentage` tends to improve the accuracy of the final results. " + + "If `visit_percentage` is set is set for `bbq_disk`, `num_candidates` is ignored. " + + "Defaults to a reasonable percent per shard of ~1% for every 1 million vectors." ), @MapParam.MapParamEntry( name = "similarity", From e02367bf8620aa4a722d27ac70d300a7a06a97c9 Mon Sep 17 00:00:00 2001 From: John Wagster Date: Thu, 28 Aug 2025 20:39:43 -0500 Subject: [PATCH 07/36] docs --- .../mapping-reference/dense-vector.md | 17 ++++++++++++----- .../query-dsl/query-dsl-knn-query.md | 6 ++++++ .../DiversifyingParentBlockQueryTests.java | 1 + 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/docs/reference/elasticsearch/mapping-reference/dense-vector.md b/docs/reference/elasticsearch/mapping-reference/dense-vector.md index 70b4296e645e3..5d45eb39b40eb 100644 --- a/docs/reference/elasticsearch/mapping-reference/dense-vector.md +++ b/docs/reference/elasticsearch/mapping-reference/dense-vector.md @@ -341,18 +341,19 @@ $$$dense-vector-index-options$$$ `type` : (Required, string) The type of kNN algorithm to use. Can be either any of: * `hnsw` - This utilizes the [HNSW algorithm](https://arxiv.org/abs/1603.09320) for scalable approximate kNN search. This supports all `element_type` values. - * `int8_hnsw` - The default index type for some float vectors: + * `int8_hnsw` - The default index type for some float vectors: * {applies_to}`stack: ga 9.1` Default for float vectors with less than 384 dimensions. * {applies_to}`stack: ga 9.0` Default for float all vectors. This utilizes the [HNSW algorithm](https://arxiv.org/abs/1603.09320) in addition to automatically scalar quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint by 4x at the cost of some accuracy. See [Automatically quantize vectors for kNN search](#dense-vector-quantization). * `int4_hnsw` - This utilizes the [HNSW algorithm](https://arxiv.org/abs/1603.09320) in addition to automatically scalar quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint by 8x at the cost of some accuracy. See [Automatically quantize vectors for kNN search](#dense-vector-quantization). * `bbq_hnsw` - This utilizes the [HNSW algorithm](https://arxiv.org/abs/1603.09320) in addition to automatically binary quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint by 32x at the cost of accuracy. See [Automatically quantize vectors for kNN search](#dense-vector-quantization). - + {applies_to}`stack: ga 9.1` `bbq_hnsw` is the default index type for float vectors with greater than or equal to 384 dimensions. * `flat` - This utilizes a brute-force search algorithm for exact kNN search. This supports all `element_type` values. - * `int8_flat` - This utilizes a brute-force search algorithm in addition to automatically scalar quantization. Only supports `element_type` of `float`. - * `int4_flat` - This utilizes a brute-force search algorithm in addition to automatically half-byte scalar quantization. Only supports `element_type` of `float`. - * `bbq_flat` - This utilizes a brute-force search algorithm in addition to automatically binary quantization. Only supports `element_type` of `float`. + * `int8_flat` - This utilizes a brute-force search algorithm in addition to automatic scalar quantization. Only supports `element_type` of `float`. + * `int4_flat` - This utilizes a brute-force search algorithm in addition to automatic half-byte scalar quantization. Only supports `element_type` of `float`. + * `bbq_flat` - This utilizes a brute-force search algorithm in addition to automatic binary quantization. Only supports `element_type` of `float`. + * {applies_to}`stack: ga 9.1` `bbq_disk` - This utilizes a clustering search algorithm in addition to automatic binary quantization. Only supports `element_type` of `float`. `m` : (Optional, integer) The number of neighbors each node will be connected to in the HNSW graph. Defaults to `16`. Only applicable to `hnsw`, `int8_hnsw`, `int4_hnsw` and `bbq_hnsw` index types. @@ -363,6 +364,12 @@ $$$dense-vector-index-options$$$ `confidence_interval` : (Optional, float) Only applicable to `int8_hnsw`, `int4_hnsw`, `int8_flat`, and `int4_flat` index types. The confidence interval to use when quantizing the vectors. Can be any value between and including `0.90` and `1.0` or exactly `0`. When the value is `0`, this indicates that dynamic quantiles should be calculated for optimized quantization. When between `0.90` and `1.0`, this value restricts the values used when calculating the quantization thresholds. For example, a value of `0.95` will only use the middle 95% of the values when calculating the quantization thresholds (e.g. the highest and lowest 2.5% of values will be ignored). Defaults to `1/(dims + 1)` for `int8` quantized vectors and `0` for `int4` for dynamic quantile calculation. +`default_visit_percentage` {applies_to}`stack: ga 9.1` +: (Optional, integer) Only applicable to `bbq_disk`. The default percentage of vectors to visit when exploring each shard. Defaults to ~1% for every 1 million vectors. + +`cluster_size` {applies_to}`stack: ga 9.1` +: (Optional, integer) Only applicable to `bbq_disk`. The number of vectors per cluster. Smaller cluster sizes increases accuracy at the cost of performance. Defaults to `384`. Must be a value between `64` and `65536`. + `rescore_vector` {applies_to}`stack: preview 9.0, ga 9.1` : (Optional, object) An optional section that configures automatic vector rescoring on knn queries for the given field. Only applicable to quantized index types. :::::{dropdown} Properties of rescore_vector diff --git a/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md b/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md index 1b41d4cccfc51..e5f902c5b2b7c 100644 --- a/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md +++ b/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md @@ -2,6 +2,9 @@ navigation_title: "Knn" mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-knn-query.html +applies_to: + stack: all + serverless: all --- # Knn query [query-dsl-knn-query] @@ -87,6 +90,9 @@ If all queried fields are of type [semantic_text](/reference/elasticsearch/mappi : (Optional, integer) The number of nearest neighbor candidates to consider per shard while doing knn search. Cannot exceed 10,000. Increasing `num_candidates` tends to improve the accuracy of the final results. Defaults to `1.5 * k` if `k` is set, or `1.5 * size` if `k` is not set. +```{applies_to} +stack: ga 9.2 +``` `visit_percentage` : (Optional, float) The percentage of vectors to explore per shard while doing knn search with `bbq_disk`. Must be between 1 and 100. 0 will default to using `num_candidates` for calculating the percent visited. Increasing `visit_percentage` tends to improve the accuracy of the final results. If `visit_percentage` is set for `bbq_disk`, `num_candidates` is ignored. Defaults to a reasonable percent per shard of ~1% for every 1 million vectors. diff --git a/server/src/test/java/org/elasticsearch/search/vectors/DiversifyingParentBlockQueryTests.java b/server/src/test/java/org/elasticsearch/search/vectors/DiversifyingParentBlockQueryTests.java index 8ff81cda6e8a0..4d9417c402700 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/DiversifyingParentBlockQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/DiversifyingParentBlockQueryTests.java @@ -116,6 +116,7 @@ public void testRandom() throws IOException { VectorData.fromFloats(queries[i]), 10, 10, + 10f, null, null, null, From e282003bdcc4c9e24b9da47168e8ef1d9c11a7a1 Mon Sep 17 00:00:00 2001 From: John Wagster Date: Thu, 28 Aug 2025 21:16:58 -0500 Subject: [PATCH 08/36] iter --- .../search/query/RescoreKnnVectorQueryIT.java | 3 ++- .../retriever/RetrieverTelemetryIT.java | 2 +- .../search/vectors/KnnSearchBuilder.java | 1 + .../search/TransportSearchActionTests.java | 4 ++-- .../vectors/DenseVectorFieldTypeTests.java | 24 ++++++++++++++++--- .../KnnRetrieverBuilderParsingTests.java | 2 ++ .../RankDocsRetrieverBuilderTests.java | 1 + 7 files changed, 30 insertions(+), 7 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java index 43aa2c9effb1f..875cbbe4de484 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java @@ -129,7 +129,7 @@ public static TestParams generate() { randomVector(numDims), k, (int) (k * randomFloatBetween(1.0f, 10.0f, true)), - (float) randomFloatBetween(0.0f, 100.0f, true), + randomFloatBetween(0.0f, 100.0f, true), new RescoreVectorBuilder(randomFloatBetween(1.0f, 100f, true)) ); } @@ -175,6 +175,7 @@ public void testKnnRetriever() { null, testParams.k, testParams.numCands, + testParams.visitPercentage, testParams.rescoreVectorBuilder, null ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverTelemetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverTelemetryIT.java index 56f1e51113bc9..0996c5e3976c0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverTelemetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverTelemetryIT.java @@ -84,7 +84,7 @@ public void testTelemetryForRetrievers() throws IOException { // search#1 - this will record 1 entry for "retriever" in `sections`, and 1 for "knn" under `retrievers` { performSearch( - new SearchSourceBuilder().retriever(new KnnRetrieverBuilder("vector", new float[] { 1.0f }, null, 10, 15, null, null)) + new SearchSourceBuilder().retriever(new KnnRetrieverBuilder("vector", new float[] { 1.0f }, null, 10, 15, 10f, null, null)) ); } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index 37b143e9fd3c6..c54abc3aae96a 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -589,6 +589,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(field); out.writeVInt(k); out.writeVInt(numCands); + out.writeFloat(visitPercentage); if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalWriteable(queryVector); } else { diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java index 9f286efe28083..8173376c0d0d0 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java @@ -1396,7 +1396,7 @@ public void testShouldMinimizeRoundtrips() throws Exception { { SearchRequest searchRequest = new SearchRequest(); SearchSourceBuilder source = new SearchSourceBuilder(); - source.knnSearch(List.of(new KnnSearchBuilder("field", new float[] { 1, 2, 3 }, 10, 50, null, null))); + source.knnSearch(List.of(new KnnSearchBuilder("field", new float[] { 1, 2, 3 }, 10, 50, 10f, null, null))); searchRequest.source(source); searchRequest.setCcsMinimizeRoundtrips(true); @@ -1411,7 +1411,7 @@ public void testAdjustSearchType() { // If the search includes kNN, we should always use DFS_QUERY_THEN_FETCH SearchRequest searchRequest = new SearchRequest(); SearchSourceBuilder source = new SearchSourceBuilder(); - source.knnSearch(List.of(new KnnSearchBuilder("field", new float[] { 1, 2, 3 }, 10, 50, null, null))); + source.knnSearch(List.of(new KnnSearchBuilder("field", new float[] { 1, 2, 3 }, 10, 50, 10f, null, null))); searchRequest.source(source); TransportSearchAction.adjustSearchType(searchRequest, randomBoolean()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 2524422ed8f90..beb9e7dd828bd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -230,6 +230,7 @@ public void testCreateNestedKnnQuery() { VectorData.fromFloats(queryVector), 10, 10, + 10f, null, null, null, @@ -269,6 +270,7 @@ public void testCreateNestedKnnQuery() { vectorData, 10, 10, + 10f, null, null, null, @@ -287,6 +289,7 @@ public void testCreateNestedKnnQuery() { vectorData, 10, 10, + 10f, null, null, null, @@ -365,6 +368,7 @@ public void testFloatCreateKnnQuery() { VectorData.fromFloats(new float[] { 0.3f, 0.1f, 1.0f, 0.0f }), 10, 10, + 10f, null, null, null, @@ -396,6 +400,7 @@ public void testFloatCreateKnnQuery() { VectorData.fromFloats(queryVector), 10, 10, + 10f, null, null, null, @@ -423,6 +428,7 @@ public void testFloatCreateKnnQuery() { VectorData.fromFloats(new float[BBQ_MIN_DIMS]), 10, 10, + 10f, null, null, null, @@ -455,6 +461,7 @@ public void testCreateKnnQueryMaxDims() { VectorData.fromFloats(queryVector), 10, 10, + 10f, null, null, null, @@ -493,6 +500,7 @@ public void testCreateKnnQueryMaxDims() { vectorData, 10, 10, + 10f, null, null, null, @@ -526,6 +534,7 @@ public void testByteCreateKnnQuery() { VectorData.fromFloats(new float[] { 0.3f, 0.1f, 1.0f }), 10, 10, + 10f, null, null, null, @@ -553,6 +562,7 @@ public void testByteCreateKnnQuery() { VectorData.fromFloats(new float[] { 0.0f, 0.0f, 0.0f }), 10, 10, + 10f, null, null, null, @@ -569,6 +579,7 @@ public void testByteCreateKnnQuery() { new VectorData(null, new byte[] { 0, 0, 0 }), 10, 10, + 10f, null, null, null, @@ -598,6 +609,7 @@ public void testRescoreOversampleUsedWithoutQuantization() { new VectorData(null, new byte[] { 1, 4, 10 }), 10, 100, + 10f, randomFloatBetween(1.0F, 10.0F, false), null, null, @@ -647,11 +659,11 @@ public void testRescoreOversampleModifiesNumCandidates() { ); // Total results is k, internal k is multiplied by oversample - checkRescoreQueryParameters(fieldType, 10, 200, 2.5F, 25, 200, 10); + checkRescoreQueryParameters(fieldType, 10, 200, 10f, 2.5F, 25, 200, 10); // If numCands < k, update numCands to k - checkRescoreQueryParameters(fieldType, 10, 20, 2.5F, 25, 25, 10); + checkRescoreQueryParameters(fieldType, 10, 20, 10f, 2.5F, 25, 25, 10); // Oversampling limits for k - checkRescoreQueryParameters(fieldType, 1000, 1000, 11.0F, OVERSAMPLE_LIMIT, OVERSAMPLE_LIMIT, 1000); + checkRescoreQueryParameters(fieldType, 1000, 1000, 10f, 11.0F, OVERSAMPLE_LIMIT, OVERSAMPLE_LIMIT, 1000); } public void testRescoreOversampleQueryOverrides() { @@ -671,6 +683,7 @@ public void testRescoreOversampleQueryOverrides() { VectorData.fromFloats(new float[] { 1, 4, 10 }), 10, 100, + 10f, 0f, null, null, @@ -700,6 +713,7 @@ public void testRescoreOversampleQueryOverrides() { VectorData.fromFloats(new float[] { 1, 4, 10 }), 10, 100, + 10f, 2f, null, null, @@ -740,6 +754,7 @@ public void testFilterSearchThreshold() { VectorData.fromFloats(new float[] { 1, 4, 10 }), 10, 100, + 10f, 0f, null, null, @@ -756,6 +771,7 @@ public void testFilterSearchThreshold() { VectorData.fromFloats(new float[] { 1, 4, 10 }), 10, 100, + 10f, 0f, null, null, @@ -776,6 +792,7 @@ private static void checkRescoreQueryParameters( DenseVectorFieldType fieldType, int k, int candidates, + float visitPercentage, float oversample, int expectedK, int expectedCandidates, @@ -785,6 +802,7 @@ private static void checkRescoreQueryParameters( VectorData.fromFloats(new float[] { 1, 4, 10 }), k, candidates, + visitPercentage, oversample, null, null, diff --git a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java index 2724b86f9acd4..4f45b2c9a473d 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java @@ -52,6 +52,7 @@ public static KnnRetrieverBuilder createRandomKnnRetrieverBuilder() { float[] vector = randomVector(dim); int k = randomIntBetween(1, 100); int numCands = randomIntBetween(k + 20, 1000); + float visitPercentage = randomFloatBetween(0.0f, 100.0f, true); Float similarity = randomBoolean() ? null : randomFloat(); RescoreVectorBuilder rescoreVectorBuilder = randomBoolean() ? null @@ -63,6 +64,7 @@ public static KnnRetrieverBuilder createRandomKnnRetrieverBuilder() { null, k, numCands, + visitPercentage, rescoreVectorBuilder, similarity ); diff --git a/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java b/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java index 165ad9b2de183..94aa7c2761470 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java @@ -70,6 +70,7 @@ private List innerRetrievers(QueryRewriteContext queryRewriteC null, randomInt(10), randomIntBetween(10, 100), + randomFloatBetween(0.0f, 100.0f, true), randomBoolean() ? null : new RescoreVectorBuilder(randomFloatBetween(1.0f, 10.0f, false)), randomFloat() ); From 98848b9e32252eb045e8ffe6c4cd82dacd29fa6c Mon Sep 17 00:00:00 2001 From: John Wagster Date: Thu, 28 Aug 2025 22:09:16 -0500 Subject: [PATCH 09/36] iter --- .../elasticsearch/search/vectors/KnnSearchBuilder.java | 9 +++++---- .../search/vectors/KnnSearchBuilderTests.java | 8 ++++---- .../search/vectors/KnnSearchRequestParserTests.java | 7 +++++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index c54abc3aae96a..1694fb7f8cfbf 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -64,11 +64,12 @@ public class KnnSearchBuilder implements Writeable, ToXContentFragment, Rewritea // TODO optimize parsing for when BYTE values are provided return new Builder().field((String) args[0]) .queryVector((VectorData) args[1]) - .queryVectorBuilder((QueryVectorBuilder) args[4]) + .queryVectorBuilder((QueryVectorBuilder) args[5]) .k((Integer) args[2]) .numCandidates((Integer) args[3]) - .similarity((Float) args[5]) - .rescoreVectorBuilder((RescoreVectorBuilder) args[6]); + .visitPercentage((Float) args[4]) + .similarity((Float) args[6]) + .rescoreVectorBuilder((RescoreVectorBuilder) args[7]); }); static { @@ -281,7 +282,7 @@ private KnnSearchBuilder( throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot exceed [" + NUM_CANDS_LIMIT + "]"); } if (visitPercentage < 0f || visitPercentage > 100f) { - throw new IllegalArgumentException("[" + VISIT_PERCENTAGE.getPreferredName() + "] must be between [0] and [100]"); + throw new IllegalArgumentException("[" + VISIT_PERCENTAGE.getPreferredName() + "] must be between 0 and 100"); } if (queryVector == null && queryVectorBuilder == null) { throw new IllegalArgumentException( diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java index eb4663d8226c9..15bee32aaabc9 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java @@ -282,17 +282,17 @@ public void testNumCandsExceedsLimit() { public void testVisitPercentageLessThan0() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> new KnnSearchBuilder("field", randomVector(3), 50, 10, -190f, null, null) + () -> new KnnSearchBuilder("field", randomVector(3), 50, 100, -190f, null, null) ); - assertThat(e.getMessage(), containsString("[visit_percentage] cannot be less than [0.0]")); + assertThat(e.getMessage(), containsString("[visit_percentage] must be between 0 and 100")); } public void testVisitPercentageGreaterThan100() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> new KnnSearchBuilder("field", randomVector(3), 100, 10002, 100000f, null, null) + () -> new KnnSearchBuilder("field", randomVector(3), 100, 1000, 100000f, null, null) ); - assertThat(e.getMessage(), containsString("[visit_percentage] cannot be greater than [100.0]")); + assertThat(e.getMessage(), containsString("[visit_percentage] must be between 0 and 100")); } public void testInvalidK() { diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java index 00ca8de3de982..3649cea2260eb 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java @@ -173,6 +173,7 @@ public void testNumCandsLessThanK() throws IOException { .field(KnnSearch.FIELD_FIELD.getPreferredName(), "field") .field(KnnSearch.K_FIELD.getPreferredName(), 100) .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), 80) + .field(KnnSearch.VISIT_PERCENTAGE_FIELD.getPreferredName(), 100.0f) .field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), new float[] { 1.0f, 2.0f, 3.0f }) .endObject() .endObject(); @@ -189,6 +190,7 @@ public void testNumCandsExceedsLimit() throws IOException { .field(KnnSearch.FIELD_FIELD.getPreferredName(), "field") .field(KnnSearch.K_FIELD.getPreferredName(), 100) .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), 10002) + .field(KnnSearch.VISIT_PERCENTAGE_FIELD.getPreferredName(), 100.0f) .field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), new float[] { 1.0f, 2.0f, 3.0f }) .endObject() .endObject(); @@ -211,7 +213,7 @@ public void testVisitPercnetageLessThan0() throws IOException { .endObject(); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseSearchRequest(builder)); - assertThat(e.getMessage(), containsString("[visit_percentage] cannot be less than [0.0f]")); + assertThat(e.getMessage(), containsString("[visit_percentage] must be between 0 and 100")); } public void testVisitPercnetageGreaterThan100() throws IOException { @@ -228,7 +230,7 @@ public void testVisitPercnetageGreaterThan100() throws IOException { .endObject(); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseSearchRequest(builder)); - assertThat(e.getMessage(), containsString("[visit_percentage] cannot be greater than [100.0f]")); + assertThat(e.getMessage(), containsString("[visit_percentage] must be between 0 and 100")); } public void testInvalidK() throws IOException { @@ -239,6 +241,7 @@ public void testInvalidK() throws IOException { .field(KnnSearch.FIELD_FIELD.getPreferredName(), "field") .field(KnnSearch.K_FIELD.getPreferredName(), 0) .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), 10) + .field(KnnSearch.VISIT_PERCENTAGE_FIELD.getPreferredName(), 100.0f) .field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), new float[] { 1.0f, 2.0f, 3.0f }) .endObject() .endObject(); From 43f161fd53e3913dda291b41709ce21edd25a3b8 Mon Sep 17 00:00:00 2001 From: John Wagster Date: Fri, 29 Aug 2025 13:59:40 -0500 Subject: [PATCH 10/36] iter --- .../search/retriever/KnnRetrieverBuilder.java | 15 +++--- .../search/vectors/KnnSearchBuilder.java | 21 ++++----- .../vectors/KnnSearchRequestParser.java | 13 +++--- .../search/vectors/KnnVectorQueryBuilder.java | 27 ++--------- ...AbstractKnnVectorQueryBuilderTestCase.java | 46 ++++++++----------- .../search/vectors/KnnSearchBuilderTests.java | 2 +- 6 files changed, 51 insertions(+), 73 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java index 8657bfa3b6afd..fb4a21b87fb7e 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java @@ -74,7 +74,7 @@ public final class KnnRetrieverBuilder extends RetrieverBuilder { (QueryVectorBuilder) args[2], (int) args[3], (int) args[4], - (float) args[5], + (Float) args[5], (RescoreVectorBuilder) args[7], (Float) args[6] ); @@ -91,7 +91,7 @@ public final class KnnRetrieverBuilder extends RetrieverBuilder { ); PARSER.declareInt(constructorArg(), K_FIELD); PARSER.declareInt(constructorArg(), NUM_CANDS_FIELD); - PARSER.declareFloat(constructorArg(), VISIT_PERCENTAGE_FIELD); + PARSER.declareFloat(optionalConstructorArg(), VISIT_PERCENTAGE_FIELD); PARSER.declareFloat(optionalConstructorArg(), VECTOR_SIMILARITY); PARSER.declareField( optionalConstructorArg(), @@ -121,7 +121,7 @@ public KnnRetrieverBuilder( QueryVectorBuilder queryVectorBuilder, int k, int numCands, - float visitPercentage, + Float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity ) { @@ -147,7 +147,7 @@ public KnnRetrieverBuilder( this.queryVectorBuilder = queryVectorBuilder; this.k = k; this.numCands = numCands; - this.visitPercentage = visitPercentage; + this.visitPercentage = visitPercentage == null ? 0.0f : visitPercentage; this.similarity = similarity; this.rescoreVectorBuilder = rescoreVectorBuilder; } @@ -269,7 +269,10 @@ public void doToXContent(XContentBuilder builder, Params params) throws IOExcept builder.field(FIELD_FIELD.getPreferredName(), field); builder.field(K_FIELD.getPreferredName(), k); builder.field(NUM_CANDS_FIELD.getPreferredName(), numCands); - builder.field(VISIT_PERCENTAGE_FIELD.getPreferredName(), visitPercentage); + + if( visitPercentage != 0.0f ) { + builder.field(VISIT_PERCENTAGE_FIELD.getPreferredName(), visitPercentage); + } if (queryVector != null) { builder.field(QUERY_VECTOR_FIELD.getPreferredName(), queryVector.get()); @@ -304,7 +307,7 @@ public boolean doEquals(Object o) { @Override public int doHashCode() { - int result = Objects.hash(field, queryVectorBuilder, k, numCands, rescoreVectorBuilder, similarity); + int result = Objects.hash(field, queryVectorBuilder, k, numCands, visitPercentage, rescoreVectorBuilder, similarity); result = 31 * result + Arrays.hashCode(queryVector != null ? queryVector.get() : null); return result; } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index 1694fb7f8cfbf..202ac4eb30674 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -49,7 +49,7 @@ public class KnnSearchBuilder implements Writeable, ToXContentFragment, Rewritea public static final ParseField FIELD_FIELD = new ParseField("field"); public static final ParseField K_FIELD = new ParseField("k"); public static final ParseField NUM_CANDS_FIELD = new ParseField("num_candidates"); - public static final ParseField VISIT_PERCENTAGE = new ParseField("visit_percentage"); + public static final ParseField VISIT_PERCENTAGE_FIELD = new ParseField("visit_percentage"); public static final ParseField QUERY_VECTOR_FIELD = new ParseField("query_vector"); public static final ParseField QUERY_VECTOR_BUILDER_FIELD = new ParseField("query_vector_builder"); public static final ParseField VECTOR_SIMILARITY = new ParseField("similarity"); @@ -82,7 +82,7 @@ public class KnnSearchBuilder implements Writeable, ToXContentFragment, Rewritea ); PARSER.declareInt(optionalConstructorArg(), K_FIELD); PARSER.declareInt(optionalConstructorArg(), NUM_CANDS_FIELD); - PARSER.declareFloat(optionalConstructorArg(), VISIT_PERCENTAGE); + PARSER.declareFloat(optionalConstructorArg(), VISIT_PERCENTAGE_FIELD); PARSER.declareNamedObject( optionalConstructorArg(), (p, c, n) -> p.namedObject(QueryVectorBuilder.class, n, c), @@ -263,7 +263,7 @@ private KnnSearchBuilder( List filterQueries, int k, int numCandidates, - float visitPercentage, + Float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity, InnerHitBuilder innerHitBuilder, @@ -282,7 +282,7 @@ private KnnSearchBuilder( throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot exceed [" + NUM_CANDS_LIMIT + "]"); } if (visitPercentage < 0f || visitPercentage > 100f) { - throw new IllegalArgumentException("[" + VISIT_PERCENTAGE.getPreferredName() + "] must be between 0 and 100"); + throw new IllegalArgumentException("[" + VISIT_PERCENTAGE_FIELD.getPreferredName() + "] must be between 0 and 100"); } if (queryVector == null && queryVectorBuilder == null) { throw new IllegalArgumentException( @@ -321,11 +321,7 @@ public KnnSearchBuilder(StreamInput in) throws IOException { this.field = in.readString(); this.k = in.readVInt(); this.numCands = in.readVInt(); - if (in.getTransportVersion().onOrAfter(TransportVersions.VISIT_PERCENTAGE)) { - this.visitPercentage = in.readFloat(); - } else { - this.visitPercentage = 0f; - } + this.visitPercentage = in.readFloat(); if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { this.queryVector = in.readOptionalWriteable(VectorData::new); } else { @@ -527,6 +523,7 @@ public int hashCode() { field, k, numCands, + visitPercentage, querySupplier, queryVectorBuilder, rescoreVectorBuilder, @@ -544,7 +541,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(FIELD_FIELD.getPreferredName(), field); builder.field(K_FIELD.getPreferredName(), k); builder.field(NUM_CANDS_FIELD.getPreferredName(), numCands); - builder.field(VISIT_PERCENTAGE.getPreferredName(), visitPercentage); + + if( visitPercentage != 0f ) { + builder.field(VISIT_PERCENTAGE_FIELD.getPreferredName(), visitPercentage); + } if (queryVectorBuilder != null) { builder.startObject(QUERY_VECTOR_BUILDER_FIELD.getPreferredName()); @@ -706,7 +706,6 @@ public KnnSearchBuilder build(int size) { int adjustedNumCandidates = numCandidates == null ? Math.round(Math.min(NUM_CANDS_LIMIT, NUM_CANDS_MULTIPLICATIVE_FACTOR * adjustedK)) : numCandidates; - // FIXME: adjust this based on the other params? float adjustedVisitPercentage = visitPercentage == null ? 0f : visitPercentage; return new KnnSearchBuilder( field, diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java index 0287285545b1f..5fd3bf3213ef3 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java @@ -31,6 +31,7 @@ import java.util.Objects; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; /** * A builder used in {@link RestKnnSearchAction} to convert the kNN REST request @@ -199,8 +200,6 @@ static class KnnSearch { static final ParseField FIELD_FIELD = new ParseField("field"); static final ParseField K_FIELD = new ParseField("k"); static final ParseField NUM_CANDS_FIELD = new ParseField("num_candidates"); - // FIXME: update docs to reflect the new optional field - // FIXME: add more tests around this field static final ParseField VISIT_PERCENTAGE_FIELD = new ParseField("visit_percentage"); static final ParseField QUERY_VECTOR_FIELD = new ParseField("query_vector"); @@ -211,7 +210,7 @@ static class KnnSearch { for (int i = 0; i < vector.size(); i++) { vectorArray[i] = vector.get(i); } - return new KnnSearch((String) args[0], vectorArray, (int) args[2], (int) args[3], (float) args[4]); + return new KnnSearch((String) args[0], vectorArray, (int) args[2], (int) args[3], (Float) args[4]); }); static { @@ -219,7 +218,7 @@ static class KnnSearch { PARSER.declareFloatArray(constructorArg(), QUERY_VECTOR_FIELD); PARSER.declareInt(constructorArg(), K_FIELD); PARSER.declareInt(constructorArg(), NUM_CANDS_FIELD); - PARSER.declareFloat(constructorArg(), VISIT_PERCENTAGE_FIELD); + PARSER.declareFloat(optionalConstructorArg(), VISIT_PERCENTAGE_FIELD); } public static KnnSearch parse(XContentParser parser) throws IOException { @@ -240,12 +239,12 @@ public static KnnSearch parse(XContentParser parser) throws IOException { * @param k the final number of nearest neighbors to return as top hits * @param numCands the number of nearest neighbor candidates to consider per shard */ - KnnSearch(String field, float[] queryVector, int k, int numCands, float visitPercentage) { + KnnSearch(String field, float[] queryVector, int k, int numCands, Float visitPercentage) { this.field = field; this.queryVector = queryVector; this.k = k; this.numCands = numCands; - this.visitPercentage = visitPercentage; + this.visitPercentage = visitPercentage == null ? 0f : visitPercentage; } public KnnVectorQueryBuilder toQueryBuilder() { @@ -282,7 +281,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - int result = Objects.hash(field, k, numCands); + int result = Objects.hash(field, k, numCands, visitPercentage); result = 31 * result + Arrays.hashCode(queryVector); return result; } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index 90c769ca86b57..40224596cb4a5 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -266,12 +266,7 @@ public KnnVectorQueryBuilder(StreamInput in) throws IOException { } else { this.numCands = in.readVInt(); } - // FIXME: validate transport version changes - if (in.getTransportVersion().onOrAfter(TransportVersions.VISIT_PERCENTAGE)) { - this.visitPercentage = in.readOptionalFloat(); - } else { - this.visitPercentage = in.readFloat(); - } + this.visitPercentage = in.readOptionalFloat(); if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { this.queryVector = in.readOptionalWriteable(VectorData::new); } else { @@ -381,21 +376,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeVInt(numCands); } } - if (out.getTransportVersion().onOrAfter(TransportVersions.VISIT_PERCENTAGE)) { - out.writeOptionalFloat(visitPercentage); - } else { - if (visitPercentage == null) { - throw new IllegalArgumentException( - "[" - + VISIT_PERCENTAGE_FIELD.getPreferredName() - + "] field was mandatory in previous releases " - + "and is required to be non-null by some nodes. " - + "Please make sure to provide the parameter as part of the request." - ); - } else { - out.writeFloat(visitPercentage); - } - } + out.writeOptionalFloat(visitPercentage); if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalWriteable(queryVector); } else { @@ -582,7 +563,7 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { int adjustedNumCands = numCands == null ? Math.round(Math.min(NUM_CANDS_MULTIPLICATIVE_FACTOR * k, NUM_CANDS_LIMIT)) : numCands; // FIXME: how do the other params interact with this? - float adjustedVisitPercentage = visitPercentage == null ? 0.0f : visitPercentage; + float adjustedVisitPercentage = visitPercentage == null ? 0f : visitPercentage; if (fieldType == null) { return new MatchNoDocsQuery(); @@ -674,6 +655,7 @@ protected int doHashCode() { Objects.hashCode(queryVector), k, numCands, + visitPercentage, filterQueries, vectorSimilarity, queryVectorBuilder, @@ -687,6 +669,7 @@ protected boolean doEquals(KnnVectorQueryBuilder other) { && Objects.equals(queryVector, other.queryVector) && Objects.equals(k, other.k) && Objects.equals(numCands, other.numCands) + && Objects.equals(visitPercentage, other.visitPercentage) && Objects.equals(filterQueries, other.filterQueries) && Objects.equals(vectorSimilarity, other.vectorSimilarity) && Objects.equals(queryVectorBuilder, other.queryVectorBuilder) diff --git a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java index 81f7d0f550c07..cbdd4d9c96719 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java @@ -347,7 +347,8 @@ public void testValidOutput() { 2.0, 3.0 ], - "num_candidates" : 10 + "num_candidates" : 10, + "visit_percentage" : 10.0 } }"""; assertEquals(expected, query.toString()); @@ -363,10 +364,27 @@ public void testValidOutput() { 3.0 ], "k" : 5, - "num_candidates" : 10 + "num_candidates" : 10, + "visit_percentage" : 10.0 } }"""; assertEquals(expected2, query2.toString()); + + KnnVectorQueryBuilder query3 = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, 5, 10, null, null, null); + String expected3 = """ + { + "knn" : { + "field" : "vector", + "query_vector" : [ + 1.0, + 2.0, + 3.0 + ], + "k" : 5, + "num_candidates" : 10 + } + }"""; + assertEquals(expected3, query3.toString()); } @Override @@ -471,30 +489,6 @@ public void testBWCVersionSerializationRescoreVector() throws IOException { assertBWCSerialization(query, queryNoRescoreVector, version); } - // FIXME: write me - public void testBWCVersionSerializationVisitPercentage() throws IOException { - KnnVectorQueryBuilder query = createTestQueryBuilder(); - TransportVersion version = TransportVersionUtils.randomVersionBetween( - random(), - TransportVersions.V_9_0_0, - TransportVersions.VISIT_PERCENTAGE - ); - VectorData vectorData = version.onOrAfter(TransportVersions.V_9_0_0) - ? query.queryVector() - : VectorData.fromFloats(query.queryVector().asFloatVector()); - Float visitPercentage = version.before(TransportVersions.VISIT_PERCENTAGE) ? null : query.visitPercentage(); - KnnVectorQueryBuilder queryVisitPercentage = new KnnVectorQueryBuilder( - query.getFieldName(), - vectorData, - query.k(), - query.numCands(), - visitPercentage, - null, - query.getVectorSimilarity() - ).queryName(query.queryName()).boost(query.boost()).addFilterQueries(query.filterQueries()); - assertBWCSerialization(query, queryVisitPercentage, version); - } - private void assertBWCSerialization(QueryBuilder newQuery, QueryBuilder bwcQuery, TransportVersion version) throws IOException { assertSerialization(bwcQuery, version); try (BytesStreamOutput output = new BytesStreamOutput()) { diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java index 15bee32aaabc9..c2555b9112db8 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java @@ -209,7 +209,7 @@ yield new KnnSearchBuilder( case 8 -> { Float newVisitPercentage = randomValueOtherThan( instance.visitPercentage, - () -> instance.visitPercentage + ESTestCase.randomFloatBetween(-100f, 100f, true) + () -> ESTestCase.randomFloatBetween(0f, 100f, true) ); yield new KnnSearchBuilder( instance.field, From 7c304df8c64e2a0fdee19541230ec9097e675229 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 29 Aug 2025 19:08:32 +0000 Subject: [PATCH 11/36] [CI] Auto commit changes from spotless --- .../org/elasticsearch/search/retriever/KnnRetrieverBuilder.java | 2 +- .../java/org/elasticsearch/search/vectors/KnnSearchBuilder.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java index fb4a21b87fb7e..6374170e1e0a4 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java @@ -270,7 +270,7 @@ public void doToXContent(XContentBuilder builder, Params params) throws IOExcept builder.field(K_FIELD.getPreferredName(), k); builder.field(NUM_CANDS_FIELD.getPreferredName(), numCands); - if( visitPercentage != 0.0f ) { + if (visitPercentage != 0.0f) { builder.field(VISIT_PERCENTAGE_FIELD.getPreferredName(), visitPercentage); } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index 202ac4eb30674..4250d9027f383 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -542,7 +542,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(K_FIELD.getPreferredName(), k); builder.field(NUM_CANDS_FIELD.getPreferredName(), numCands); - if( visitPercentage != 0f ) { + if (visitPercentage != 0f) { builder.field(VISIT_PERCENTAGE_FIELD.getPreferredName(), visitPercentage); } From f305de58924723f3275e8575475a08b250f007db Mon Sep 17 00:00:00 2001 From: John Wagster Date: Fri, 29 Aug 2025 14:34:37 -0500 Subject: [PATCH 12/36] iter --- .../elasticsearch/mapping-reference/dense-vector.md | 6 +++--- .../src/main/java/org/elasticsearch/TransportVersions.java | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/reference/elasticsearch/mapping-reference/dense-vector.md b/docs/reference/elasticsearch/mapping-reference/dense-vector.md index 5d45eb39b40eb..a1b0998b6dfe8 100644 --- a/docs/reference/elasticsearch/mapping-reference/dense-vector.md +++ b/docs/reference/elasticsearch/mapping-reference/dense-vector.md @@ -353,7 +353,7 @@ $$$dense-vector-index-options$$$ * `int8_flat` - This utilizes a brute-force search algorithm in addition to automatic scalar quantization. Only supports `element_type` of `float`. * `int4_flat` - This utilizes a brute-force search algorithm in addition to automatic half-byte scalar quantization. Only supports `element_type` of `float`. * `bbq_flat` - This utilizes a brute-force search algorithm in addition to automatic binary quantization. Only supports `element_type` of `float`. - * {applies_to}`stack: ga 9.1` `bbq_disk` - This utilizes a clustering search algorithm in addition to automatic binary quantization. Only supports `element_type` of `float`. + * {applies_to}`stack: ga 9.2` `bbq_disk` - This utilizes a clustering search algorithm in addition to automatic binary quantization. Only supports `element_type` of `float`. `m` : (Optional, integer) The number of neighbors each node will be connected to in the HNSW graph. Defaults to `16`. Only applicable to `hnsw`, `int8_hnsw`, `int4_hnsw` and `bbq_hnsw` index types. @@ -364,10 +364,10 @@ $$$dense-vector-index-options$$$ `confidence_interval` : (Optional, float) Only applicable to `int8_hnsw`, `int4_hnsw`, `int8_flat`, and `int4_flat` index types. The confidence interval to use when quantizing the vectors. Can be any value between and including `0.90` and `1.0` or exactly `0`. When the value is `0`, this indicates that dynamic quantiles should be calculated for optimized quantization. When between `0.90` and `1.0`, this value restricts the values used when calculating the quantization thresholds. For example, a value of `0.95` will only use the middle 95% of the values when calculating the quantization thresholds (e.g. the highest and lowest 2.5% of values will be ignored). Defaults to `1/(dims + 1)` for `int8` quantized vectors and `0` for `int4` for dynamic quantile calculation. -`default_visit_percentage` {applies_to}`stack: ga 9.1` +`default_visit_percentage` {applies_to}`stack: ga 9.2` : (Optional, integer) Only applicable to `bbq_disk`. The default percentage of vectors to visit when exploring each shard. Defaults to ~1% for every 1 million vectors. -`cluster_size` {applies_to}`stack: ga 9.1` +`cluster_size` {applies_to}`stack: ga 9.2` : (Optional, integer) Only applicable to `bbq_disk`. The number of vectors per cluster. Smaller cluster sizes increases accuracy at the cost of performance. Defaults to `384`. Must be a value between `64` and `65536`. `rescore_vector` {applies_to}`stack: preview 9.0, ga 9.1` diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 292339caae913..da61d44264571 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -357,7 +357,6 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_SAMPLE_OPERATOR_STATUS = def(9_127_0_00); public static final TransportVersion ALLOCATION_DECISION_NOT_PREFERRED = def(9_145_0_00); public static final TransportVersion ESQL_QUALIFIERS_IN_ATTRIBUTES = def(9_146_0_00); - public static final TransportVersion VISIT_PERCENTAGE = def(9_147_0_00); /* * STOP! READ THIS FIRST! No, really, From b4121733ca6299ebef4cb261c5b10ffd6916c037 Mon Sep 17 00:00:00 2001 From: John Wagster Date: Fri, 29 Aug 2025 14:55:51 -0500 Subject: [PATCH 13/36] iter --- .../TextSimilarityRankRetrieverTelemetryTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverTelemetryTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverTelemetryTests.java index 82c11dcf8955d..32593ccd622db 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverTelemetryTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverTelemetryTests.java @@ -101,7 +101,7 @@ public void testTelemetryForRRFRetriever() throws IOException { // search#1 - this will record 1 entry for "retriever" in `sections`, and 1 for "knn" under `retrievers` { performSearch( - new SearchSourceBuilder().retriever(new KnnRetrieverBuilder("vector", new float[] { 1.0f }, null, 10, 15, null, null)) + new SearchSourceBuilder().retriever(new KnnRetrieverBuilder("vector", new float[] { 1.0f }, null, 10, 15, 10f, null, null)) ); } From 65af05a4b63279557938bd24b85306e9c0c5e3ed Mon Sep 17 00:00:00 2001 From: John Wagster Date: Fri, 29 Aug 2025 21:11:19 -0500 Subject: [PATCH 14/36] iter --- .../rest-apis/retrievers/knn-retriever.md | 9 +++++++++ .../search/vectors/KnnSearchBuilder.java | 6 +++--- .../search/vectors/KnnSearchRequestParser.java | 2 +- .../search/vectors/KnnVectorQueryBuilder.java | 2 +- .../xpack/esql/expression/function/vector/Knn.java | 12 ------------ .../xpack/esql/querydsl/query/KnnQuery.java | 5 ++--- .../optimizer/LocalPhysicalPlanOptimizerTests.java | 2 +- .../rank/linear/LinearRetrieverBuilderTests.java | 2 ++ 8 files changed, 19 insertions(+), 21 deletions(-) diff --git a/docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md b/docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md index 12da522214383..066a88b265665 100644 --- a/docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md +++ b/docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md @@ -41,6 +41,15 @@ A kNN retriever returns top documents from a [k-nearest neighbor search (kNN)](d The number of nearest neighbor candidates to consider per shard. Needs to be greater than `k`, or `size` if `k` is omitted, and cannot exceed 10,000. {{es}} collects `num_candidates` results from each shard, then merges them to find the top `k` results. Increasing `num_candidates` tends to improve the accuracy of the final `k` results. Defaults to `Math.min(1.5 * k, 10_000)`. +```{applies_to} +stack: ga 9.2 +``` +`visit_percentage` +: (Optional, float) + + The percentage of vectors to explore per shard while doing knn search with `bbq_disk`. Must be between 1 and 100. 0 will default to using `num_candidates` for calculating the percent visited. Increasing `visit_percentage` tends to improve the accuracy of the final results. If `visit_percentage` is set for `bbq_disk`, `num_candidates` is ignored. Defaults to a reasonable percent per shard of ~1% for every 1 million vectors. + + `filter` : (Optional, [query object or list of query objects](/reference/query-languages/querydsl.md)) diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index 4250d9027f383..70436d9aaa125 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -281,7 +281,7 @@ private KnnSearchBuilder( if (numCandidates > NUM_CANDS_LIMIT) { throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot exceed [" + NUM_CANDS_LIMIT + "]"); } - if (visitPercentage < 0f || visitPercentage > 100f) { + if (visitPercentage < 0.0f || visitPercentage > 100.0f) { throw new IllegalArgumentException("[" + VISIT_PERCENTAGE_FIELD.getPreferredName() + "] must be between 0 and 100"); } if (queryVector == null && queryVectorBuilder == null) { @@ -542,7 +542,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(K_FIELD.getPreferredName(), k); builder.field(NUM_CANDS_FIELD.getPreferredName(), numCands); - if (visitPercentage != 0f) { + if (visitPercentage != 0.0f) { builder.field(VISIT_PERCENTAGE_FIELD.getPreferredName(), visitPercentage); } @@ -706,7 +706,7 @@ public KnnSearchBuilder build(int size) { int adjustedNumCandidates = numCandidates == null ? Math.round(Math.min(NUM_CANDS_LIMIT, NUM_CANDS_MULTIPLICATIVE_FACTOR * adjustedK)) : numCandidates; - float adjustedVisitPercentage = visitPercentage == null ? 0f : visitPercentage; + float adjustedVisitPercentage = visitPercentage == null ? 0.0f : visitPercentage; return new KnnSearchBuilder( field, queryVectorBuilder, diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java index 5fd3bf3213ef3..09a098c690730 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java @@ -244,7 +244,7 @@ public static KnnSearch parse(XContentParser parser) throws IOException { this.queryVector = queryVector; this.k = k; this.numCands = numCands; - this.visitPercentage = visitPercentage == null ? 0f : visitPercentage; + this.visitPercentage = visitPercentage == null ? 0.0f : visitPercentage; } public KnnVectorQueryBuilder toQueryBuilder() { diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index 40224596cb4a5..d533c0d64e0eb 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -563,7 +563,7 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { int adjustedNumCands = numCands == null ? Math.round(Math.min(NUM_CANDS_MULTIPLICATIVE_FACTOR * k, NUM_CANDS_LIMIT)) : numCands; // FIXME: how do the other params interact with this? - float adjustedVisitPercentage = visitPercentage == null ? 0f : visitPercentage; + float adjustedVisitPercentage = visitPercentage == null ? 0.0f : visitPercentage; if (fieldType == null) { return new MatchNoDocsQuery(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java index 0141fcc5b881f..0b64fb43909df 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java @@ -57,7 +57,6 @@ import static org.elasticsearch.search.vectors.KnnVectorQueryBuilder.K_FIELD; import static org.elasticsearch.search.vectors.KnnVectorQueryBuilder.NUM_CANDS_FIELD; import static org.elasticsearch.search.vectors.KnnVectorQueryBuilder.VECTOR_SIMILARITY_FIELD; -import static org.elasticsearch.search.vectors.KnnVectorQueryBuilder.VISIT_PERCENTAGE_FIELD; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FOURTH; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -85,7 +84,6 @@ public class Knn extends FullTextFunction implements OptionalArgument, VectorFun public static final Map ALLOWED_OPTIONS = Map.ofEntries( entry(NUM_CANDS_FIELD.getPreferredName(), INTEGER), - entry(VISIT_PERCENTAGE_FIELD.getPreferredName(), FLOAT), entry(VECTOR_SIMILARITY_FIELD.getPreferredName(), FLOAT), entry(BOOST_FIELD.getPreferredName(), FLOAT), entry(KnnQuery.RESCORE_OVERSAMPLE_FIELD, FLOAT) @@ -132,16 +130,6 @@ public Knn( + "Cannot exceed 10,000. Increasing num_candidates tends to improve the accuracy of the final results. " + "Defaults to 1.5 * k" ), - @MapParam.MapParamEntry( - name = "visit_percentage", - type = "float", - valueHint = { "10.0" }, - description = "(Optional, float) The percentage of vectors to explore per shard while doing knn search " - + "with `bbq_disk`. Must be between 1 and 100. 0 will default to using `num_candidates` for calculating " - + "the percent visited. Increasing `visit_percentage` tends to improve the accuracy of the final results. " - + "If `visit_percentage` is set is set for `bbq_disk`, `num_candidates` is ignored. " - + "Defaults to a reasonable percent per shard of ~1% for every 1 million vectors." - ), @MapParam.MapParamEntry( name = "similarity", type = "double", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java index feec90a808e81..57d41cd35f10c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java @@ -23,7 +23,6 @@ import static org.elasticsearch.search.vectors.KnnVectorQueryBuilder.K_FIELD; import static org.elasticsearch.search.vectors.KnnVectorQueryBuilder.NUM_CANDS_FIELD; import static org.elasticsearch.search.vectors.KnnVectorQueryBuilder.VECTOR_SIMILARITY_FIELD; -import static org.elasticsearch.search.vectors.KnnVectorQueryBuilder.VISIT_PERCENTAGE_FIELD; public class KnnQuery extends Query { @@ -47,7 +46,6 @@ public KnnQuery(Source source, String field, float[] query, Map protected QueryBuilder asBuilder() { Integer k = (Integer) options.get(K_FIELD.getPreferredName()); Integer numCands = (Integer) options.get(NUM_CANDS_FIELD.getPreferredName()); - Float visitPercentage = (Float) options.get(VISIT_PERCENTAGE_FIELD.getPreferredName()); RescoreVectorBuilder rescoreVectorBuilder = null; Float oversample = (Float) options.get(RESCORE_OVERSAMPLE_FIELD); if (oversample != null) { @@ -55,12 +53,13 @@ protected QueryBuilder asBuilder() { } Float vectorSimilarity = (Float) options.get(VECTOR_SIMILARITY_FIELD.getPreferredName()); + // TODO: expose visit_percentage in ESQL KnnVectorQueryBuilder queryBuilder = new KnnVectorQueryBuilder( field, query, k, numCands, - visitPercentage, + null, rescoreVectorBuilder, vectorSimilarity ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index feba5628791f9..0ff102bd4307c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -1394,7 +1394,7 @@ public void testKnnOptionsPushDown() { new float[] { 0.1f, 0.2f, 0.3f }, 5, 10, - 10f, + null, new RescoreVectorBuilder(7), 0.001f ).boost(3.5f); diff --git a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverBuilderTests.java b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverBuilderTests.java index 8264ccdb22e38..b3bbae70cd9ed 100644 --- a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverBuilderTests.java +++ b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverBuilderTests.java @@ -336,6 +336,7 @@ public void testTopLevelNormalizerWithRetrieversArray() { null, 10, 100, + 10f, null, null ); @@ -365,6 +366,7 @@ public void testTopLevelNormalizerWithPerRetrieverOverrides() { null, 10, 100, + 10f, null, null ); From 8f373ce1af9fdbbfaf396efb330d592b5b1fe69c Mon Sep 17 00:00:00 2001 From: John Wagster Date: Sat, 30 Aug 2025 21:48:06 -0500 Subject: [PATCH 15/36] iter --- .../src/main/java/org/elasticsearch/TransportVersions.java | 1 + .../org/elasticsearch/search/vectors/KnnSearchBuilder.java | 6 +++++- .../elasticsearch/search/vectors/KnnVectorQueryBuilder.java | 6 +++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 1abb8177e5d74..5bf9f5ca31e86 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -354,6 +354,7 @@ static TransportVersion def(int id) { public static final TransportVersion SHARD_WRITE_LOAD_IN_CLUSTER_INFO = def(9_126_0_00); public static final TransportVersion ESQL_SAMPLE_OPERATOR_STATUS = def(9_127_0_00); public static final TransportVersion PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY = def(9_147_0_00); + public static final TransportVersion VISIT_PERCENTAGE = def(9_148_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index 70436d9aaa125..6850dc3f72500 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -321,7 +321,11 @@ public KnnSearchBuilder(StreamInput in) throws IOException { this.field = in.readString(); this.k = in.readVInt(); this.numCands = in.readVInt(); - this.visitPercentage = in.readFloat(); + if (in.getTransportVersion().onOrAfter(TransportVersions.VISIT_PERCENTAGE)) { + this.visitPercentage = in.readFloat(); + } else { + this.visitPercentage = 0.0f; + } if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { this.queryVector = in.readOptionalWriteable(VectorData::new); } else { diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index d533c0d64e0eb..699f21ace6fcd 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -266,7 +266,11 @@ public KnnVectorQueryBuilder(StreamInput in) throws IOException { } else { this.numCands = in.readVInt(); } - this.visitPercentage = in.readOptionalFloat(); + if (in.getTransportVersion().onOrAfter(TransportVersions.VISIT_PERCENTAGE)) { + this.visitPercentage = in.readOptionalFloat(); + } else { + this.visitPercentage = null; + } if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { this.queryVector = in.readOptionalWriteable(VectorData::new); } else { From 1af3fe5773646196b759d817625ca92195530322 Mon Sep 17 00:00:00 2001 From: John Wagster Date: Sat, 30 Aug 2025 22:47:17 -0500 Subject: [PATCH 16/36] iter --- .../elasticsearch/search/vectors/KnnSearchBuilder.java | 4 +++- .../search/vectors/KnnVectorQueryBuilder.java | 4 +++- .../vectors/AbstractKnnVectorQueryBuilderTestCase.java | 8 ++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index 6850dc3f72500..fa9401b4d7bc1 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -594,7 +594,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(field); out.writeVInt(k); out.writeVInt(numCands); - out.writeFloat(visitPercentage); + if (out.getTransportVersion().onOrAfter(TransportVersions.VISIT_PERCENTAGE)) { + out.writeFloat(visitPercentage); + } if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalWriteable(queryVector); } else { diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index 699f21ace6fcd..8cbace0be0678 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -380,7 +380,9 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeVInt(numCands); } } - out.writeOptionalFloat(visitPercentage); + if (out.getTransportVersion().onOrAfter(TransportVersions.VISIT_PERCENTAGE)) { + out.writeOptionalFloat(visitPercentage); + } if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalWriteable(queryVector); } else { diff --git a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java index cbdd4d9c96719..2d0a9aaf6e641 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java @@ -418,7 +418,7 @@ public void testBWCVersionSerializationFilters() throws IOException { vectorData, null, query.numCands(), - query.visitPercentage(), + null, null, null ).queryName(query.queryName()).boost(query.boost()); @@ -438,7 +438,7 @@ public void testBWCVersionSerializationSimilarity() throws IOException { vectorData, null, query.numCands(), - query.visitPercentage(), + null, null, null ).queryName(query.queryName()).boost(query.boost()).addFilterQueries(query.filterQueries()); @@ -459,7 +459,7 @@ public void testBWCVersionSerializationQuery() throws IOException { vectorData, null, query.numCands(), - query.visitPercentage(), + null, null, similarity ).queryName(query.queryName()).boost(query.boost()).addFilterQueries(query.filterQueries()); @@ -482,7 +482,7 @@ public void testBWCVersionSerializationRescoreVector() throws IOException { vectorData, k, query.numCands(), - query.visitPercentage(), + null, null, query.getVectorSimilarity() ).queryName(query.queryName()).boost(query.boost()).addFilterQueries(query.filterQueries()); From 3df33c3a26727d394949c6a4bf5667333dca683d Mon Sep 17 00:00:00 2001 From: John Wagster Date: Sun, 31 Aug 2025 14:40:26 -0500 Subject: [PATCH 17/36] iter --- .../search/retriever/KnnRetrieverBuilder.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java index 6374170e1e0a4..e570128595cdc 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java @@ -115,6 +115,20 @@ public static KnnRetrieverBuilder fromXContent(XContentParser parser, RetrieverP private final RescoreVectorBuilder rescoreVectorBuilder; private final Float similarity; + // FIXME: add ctors like this to the other builders + // FIXME: from a testing standpoint should we replace this with the ctor below sometimes + public KnnRetrieverBuilder( + String field, + float[] queryVector, + QueryVectorBuilder queryVectorBuilder, + int k, + int numCands, + RescoreVectorBuilder rescoreVectorBuilder, + Float similarity + ) { + this(field, queryVector, queryVectorBuilder, k, numCands, null, rescoreVectorBuilder, similarity); + } + public KnnRetrieverBuilder( String field, float[] queryVector, From 0b637787bfb63fdba49d66902f634a2466c0f637 Mon Sep 17 00:00:00 2001 From: John Wagster Date: Sun, 31 Aug 2025 22:19:08 -0500 Subject: [PATCH 18/36] cleanup --- .../elasticsearch/search/retriever/KnnRetrieverBuilder.java | 2 -- .../org/elasticsearch/search/vectors/KnnSearchBuilder.java | 3 +++ .../elasticsearch/search/vectors/KnnVectorQueryBuilder.java | 2 -- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java index e570128595cdc..2690a457045a2 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java @@ -115,8 +115,6 @@ public static KnnRetrieverBuilder fromXContent(XContentParser parser, RetrieverP private final RescoreVectorBuilder rescoreVectorBuilder; private final Float similarity; - // FIXME: add ctors like this to the other builders - // FIXME: from a testing standpoint should we replace this with the ctor below sometimes public KnnRetrieverBuilder( String field, float[] queryVector, diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index fa9401b4d7bc1..2c7af5d54c7c6 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -136,6 +136,7 @@ public static KnnSearchBuilder.Builder fromXContent(XContentParser parser) throw * @param queryVector the query vector * @param k the final number of nearest neighbors to return as top hits * @param numCands the number of nearest neighbor candidates to consider per shard + * @param visitPercentage percentage of the total number of vectors to visit per shard * @param rescoreVectorBuilder rescore vector information */ public KnnSearchBuilder( @@ -166,6 +167,7 @@ public KnnSearchBuilder( * @param queryVector the query vector * @param k the final number of nearest neighbors to return as top hits * @param numCands the number of nearest neighbor candidates to consider per shard + * @param visitPercentage percentage of the total number of vectors to visit per shard */ public KnnSearchBuilder( String field, @@ -186,6 +188,7 @@ public KnnSearchBuilder( * @param queryVectorBuilder the query vector builder * @param k the final number of nearest neighbors to return as top hits * @param numCands the number of nearest neighbor candidates to consider per shard + * @param visitPercentage percentage of the total number of vectors to visit per shard */ public KnnSearchBuilder( String field, diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index 8cbace0be0678..8f76e2eab370a 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -567,8 +567,6 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { } } int adjustedNumCands = numCands == null ? Math.round(Math.min(NUM_CANDS_MULTIPLICATIVE_FACTOR * k, NUM_CANDS_LIMIT)) : numCands; - - // FIXME: how do the other params interact with this? float adjustedVisitPercentage = visitPercentage == null ? 0.0f : visitPercentage; if (fieldType == null) { From 056d914374461ef0fa29fd9450bde49ef2a48bf5 Mon Sep 17 00:00:00 2001 From: John Wagster Date: Sun, 31 Aug 2025 23:14:12 -0500 Subject: [PATCH 19/36] added ivf_format feature flag --- .../index/mapper/vectors/DenseVectorFieldMapper.java | 4 ---- .../elasticsearch/search/retriever/KnnRetrieverBuilder.java | 5 ++++- .../org/elasticsearch/search/vectors/KnnSearchBuilder.java | 5 ++++- .../elasticsearch/search/vectors/KnnSearchRequestParser.java | 5 ++++- .../elasticsearch/search/vectors/KnnVectorQueryBuilder.java | 5 ++++- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 22abb1c4eca0e..70251b3b0898d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2552,7 +2552,6 @@ public Query createKnnQuery( queryVector.asByteVector(), k, numCands, - visitPercentage, filter, similarityThreshold, parentFilter, @@ -2575,7 +2574,6 @@ public Query createKnnQuery( queryVector.asByteVector(), k, numCands, - visitPercentage, filter, similarityThreshold, parentFilter, @@ -2597,7 +2595,6 @@ private Query createKnnBitQuery( byte[] queryVector, int k, int numCands, - float visitPercentage, Query filter, Float similarityThreshold, BitSetProducer parentFilter, @@ -2637,7 +2634,6 @@ private Query createKnnByteQuery( byte[] queryVector, int k, int numCands, - float visitPercentage, Query filter, Float similarityThreshold, BitSetProducer parentFilter, diff --git a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java index 2690a457045a2..a8700c7d284cb 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java @@ -34,6 +34,7 @@ import java.util.function.Supplier; import static org.elasticsearch.common.Strings.format; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.IVF_FORMAT; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; @@ -91,7 +92,9 @@ public final class KnnRetrieverBuilder extends RetrieverBuilder { ); PARSER.declareInt(constructorArg(), K_FIELD); PARSER.declareInt(constructorArg(), NUM_CANDS_FIELD); - PARSER.declareFloat(optionalConstructorArg(), VISIT_PERCENTAGE_FIELD); + if (IVF_FORMAT.isEnabled()) { + PARSER.declareFloat(optionalConstructorArg(), VISIT_PERCENTAGE_FIELD); + } PARSER.declareFloat(optionalConstructorArg(), VECTOR_SIMILARITY); PARSER.declareField( optionalConstructorArg(), diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index 2c7af5d54c7c6..b0940dc946379 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -34,6 +34,7 @@ import static org.elasticsearch.TransportVersions.V_8_11_X; import static org.elasticsearch.common.Strings.format; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.IVF_FORMAT; import static org.elasticsearch.index.query.AbstractQueryBuilder.DEFAULT_BOOST; import static org.elasticsearch.search.SearchService.DEFAULT_SIZE; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; @@ -82,7 +83,9 @@ public class KnnSearchBuilder implements Writeable, ToXContentFragment, Rewritea ); PARSER.declareInt(optionalConstructorArg(), K_FIELD); PARSER.declareInt(optionalConstructorArg(), NUM_CANDS_FIELD); - PARSER.declareFloat(optionalConstructorArg(), VISIT_PERCENTAGE_FIELD); + if (IVF_FORMAT.isEnabled()) { + PARSER.declareFloat(optionalConstructorArg(), VISIT_PERCENTAGE_FIELD); + } PARSER.declareNamedObject( optionalConstructorArg(), (p, c, n) -> p.namedObject(QueryVectorBuilder.class, n, c), diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java index 09a098c690730..585f12b36b8bf 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Objects; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.IVF_FORMAT; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; @@ -218,7 +219,9 @@ static class KnnSearch { PARSER.declareFloatArray(constructorArg(), QUERY_VECTOR_FIELD); PARSER.declareInt(constructorArg(), K_FIELD); PARSER.declareInt(constructorArg(), NUM_CANDS_FIELD); - PARSER.declareFloat(optionalConstructorArg(), VISIT_PERCENTAGE_FIELD); + if (IVF_FORMAT.isEnabled()) { + PARSER.declareFloat(optionalConstructorArg(), VISIT_PERCENTAGE_FIELD); + } } public static KnnSearch parse(XContentParser parser) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index 8f76e2eab370a..7bd3edc2e62c5 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -48,6 +48,7 @@ import static org.elasticsearch.TransportVersions.KNN_QUERY_RESCORE_OVERSAMPLE; import static org.elasticsearch.common.Strings.format; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.IVF_FORMAT; import static org.elasticsearch.search.SearchService.DEFAULT_SIZE; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; @@ -96,7 +97,9 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder Date: Sun, 31 Aug 2025 23:58:01 -0500 Subject: [PATCH 20/36] added some yaml tests only as parsing validation --- .../search.retrievers/20_knn_retriever.yml | 21 ++++++++++++++++ .../135_knn_query_nested_search_ivf.yml | 24 +++++++++++++++++++ .../search.vectors/46_knn_search_bbq_ivf.yml | 22 +++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml index 1f07884c9fadf..c00640753b0e7 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml @@ -53,6 +53,27 @@ setup: - match: {hits.hits.1._id: "3"} - match: {hits.hits.1.fields.name.0: "rabbit.jpg"} +--- +"kNN retrieve with visit_percentage": + - do: + search: + index: index1 + body: + fields: [ "name" ] + retriever: + knn: + field: vector + query_vector: [2, 2, 2, 2, 3] + k: 2 + num_candidates: 3 + visit_percentage: 1.0 + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} + + - match: {hits.hits.1._id: "3"} + - match: {hits.hits.1.fields.name.0: "rabbit.jpg"} + --- "kNN retriever with filter": - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/135_knn_query_nested_search_ivf.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/135_knn_query_nested_search_ivf.yml index fe19a9b8578fb..4fc69240aed9b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/135_knn_query_nested_search_ivf.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/135_knn_query_nested_search_ivf.yml @@ -191,3 +191,27 @@ setup: - match: {hits.hits.0._id: "3"} - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} - match: { hits.hits.0.inner_hits.nested.hits.hits.0.fields.nested.0.paragraph_id.0: "0" } + +--- +"nested kNN search works with visit_percentage": + - do: + search: + index: test + body: + fields: [ "name" ] + query: + nested: + path: nested + query: + knn: + field: nested.vector + query_vector: [-0.5, 90, -10, 14.8, -156] + num_candidates: 3 + visit_percentage: 1.0 + - match: {hits.total.value: 3} + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} + + - match: {hits.hits.1._id: "3"} + - match: {hits.hits.1.fields.name.0: "rabbit.jpg"} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/46_knn_search_bbq_ivf.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/46_knn_search_bbq_ivf.yml index 3ce9232fc4ecd..3fbbbc3de1f48 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/46_knn_search_bbq_ivf.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/46_knn_search_bbq_ivf.yml @@ -106,6 +106,28 @@ setup: - match: { hits.hits.1._id: "3" } - match: { hits.hits.2._id: "2" } --- +"Test knn search with visit_percentage": + - do: + search: + index: bbq_disk + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + visit_percentage: 1.0 + + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.2._id: "2" } +--- "Vector rescoring has same scoring as exact search for kNN section": - skip: features: "headers" From 341d49102b420f251bf42cf84fe7329d9762600f Mon Sep 17 00:00:00 2001 From: John Wagster Date: Mon, 1 Sep 2025 21:37:42 -0500 Subject: [PATCH 21/36] fix transport versions --- server/src/main/java/org/elasticsearch/TransportVersions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 4247af118d957..420fe7ebede2d 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -354,7 +354,7 @@ static TransportVersion def(int id) { public static final TransportVersion SHARD_WRITE_LOAD_IN_CLUSTER_INFO = def(9_126_0_00); public static final TransportVersion ESQL_SAMPLE_OPERATOR_STATUS = def(9_127_0_00); public static final TransportVersion PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY = def(9_147_0_00); - public static final TransportVersion STREAMS_ENDPOINT_PARAM_RESTRICTIONS = def(9_148_00_00); + public static final TransportVersion STREAMS_ENDPOINT_PARAM_RESTRICTIONS = def(9_148_0_00); public static final TransportVersion VISIT_PERCENTAGE = def(9_149_0_00); /* From 7006315d97c25aa98bb9d2d6702438f31a1b82f3 Mon Sep 17 00:00:00 2001 From: John Wagster Date: Mon, 1 Sep 2025 21:49:13 -0500 Subject: [PATCH 22/36] add feature flag check --- .../rest-api-spec/test/search.retrievers/20_knn_retriever.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml index c00640753b0e7..98713cf570eea 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml @@ -55,6 +55,9 @@ setup: --- "kNN retrieve with visit_percentage": + - requires: + cluster_features: "mapper.bbq_disk_support" + reason: 'bbq disk support required' - do: search: index: index1 From 5332d685ead5f00afd639b2bd08a3e993dd50611 Mon Sep 17 00:00:00 2001 From: John Wagster Date: Mon, 1 Sep 2025 22:26:50 -0500 Subject: [PATCH 23/36] Update docs/changelog/133753.yaml --- docs/changelog/133753.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/133753.yaml diff --git a/docs/changelog/133753.yaml b/docs/changelog/133753.yaml new file mode 100644 index 0000000000000..3eb4a93b62363 --- /dev/null +++ b/docs/changelog/133753.yaml @@ -0,0 +1,5 @@ +pr: 133753 +summary: KNN Query `visit_percentage` +area: Vector Search +type: feature +issues: [] From ae3e5668ae27fda52161921ee8862bb8dcca8724 Mon Sep 17 00:00:00 2001 From: John Wagster Date: Tue, 2 Sep 2025 09:02:41 -0500 Subject: [PATCH 24/36] Update docs/reference/query-languages/query-dsl/query-dsl-knn-query.md Co-authored-by: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com> --- docs/reference/query-languages/query-dsl/query-dsl-knn-query.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md b/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md index e5f902c5b2b7c..3105aea743ca6 100644 --- a/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md +++ b/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md @@ -94,7 +94,7 @@ If all queried fields are of type [semantic_text](/reference/elasticsearch/mappi stack: ga 9.2 ``` `visit_percentage` -: (Optional, float) The percentage of vectors to explore per shard while doing knn search with `bbq_disk`. Must be between 1 and 100. 0 will default to using `num_candidates` for calculating the percent visited. Increasing `visit_percentage` tends to improve the accuracy of the final results. If `visit_percentage` is set for `bbq_disk`, `num_candidates` is ignored. Defaults to a reasonable percent per shard of ~1% for every 1 million vectors. +: (Optional, float) The percentage of vectors to explore per shard while doing knn search with `bbq_disk`. Must be between 1 and 100. 0 will default to using `num_candidates` for calculating the percent visited. Increasing `visit_percentage` tends to improve the accuracy of the final results. If `visit_percentage` is set for `bbq_disk`, `num_candidates` is ignored. Defaults to 1% per shard for every 1 million vectors. `filter` From 6ea6c592e5a0feaa924b3c9ea30b948c25fd7bed Mon Sep 17 00:00:00 2001 From: John Wagster Date: Tue, 2 Sep 2025 09:31:32 -0500 Subject: [PATCH 25/36] consistent docs with fix to reference the range starting at 0 --- docs/reference/elasticsearch/mapping-reference/dense-vector.md | 2 +- .../elasticsearch/rest-apis/retrievers/knn-retriever.md | 2 +- docs/reference/query-languages/query-dsl/query-dsl-knn-query.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/elasticsearch/mapping-reference/dense-vector.md b/docs/reference/elasticsearch/mapping-reference/dense-vector.md index a1b0998b6dfe8..ec75e38b80946 100644 --- a/docs/reference/elasticsearch/mapping-reference/dense-vector.md +++ b/docs/reference/elasticsearch/mapping-reference/dense-vector.md @@ -365,7 +365,7 @@ $$$dense-vector-index-options$$$ : (Optional, float) Only applicable to `int8_hnsw`, `int4_hnsw`, `int8_flat`, and `int4_flat` index types. The confidence interval to use when quantizing the vectors. Can be any value between and including `0.90` and `1.0` or exactly `0`. When the value is `0`, this indicates that dynamic quantiles should be calculated for optimized quantization. When between `0.90` and `1.0`, this value restricts the values used when calculating the quantization thresholds. For example, a value of `0.95` will only use the middle 95% of the values when calculating the quantization thresholds (e.g. the highest and lowest 2.5% of values will be ignored). Defaults to `1/(dims + 1)` for `int8` quantized vectors and `0` for `int4` for dynamic quantile calculation. `default_visit_percentage` {applies_to}`stack: ga 9.2` -: (Optional, integer) Only applicable to `bbq_disk`. The default percentage of vectors to visit when exploring each shard. Defaults to ~1% for every 1 million vectors. +: (Optional, integer) Only applicable to `bbq_disk`. Must be between 0 and 100. 0 will default to using `num_candidates` for calculating the percent visited. Increasing `default_visit_percentage` tends to improve the accuracy of the final results. Defaults to ~1% per shard for every 1 million vectors. `cluster_size` {applies_to}`stack: ga 9.2` : (Optional, integer) Only applicable to `bbq_disk`. The number of vectors per cluster. Smaller cluster sizes increases accuracy at the cost of performance. Defaults to `384`. Must be a value between `64` and `65536`. diff --git a/docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md b/docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md index 066a88b265665..2b7f162255035 100644 --- a/docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md +++ b/docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md @@ -47,7 +47,7 @@ stack: ga 9.2 `visit_percentage` : (Optional, float) - The percentage of vectors to explore per shard while doing knn search with `bbq_disk`. Must be between 1 and 100. 0 will default to using `num_candidates` for calculating the percent visited. Increasing `visit_percentage` tends to improve the accuracy of the final results. If `visit_percentage` is set for `bbq_disk`, `num_candidates` is ignored. Defaults to a reasonable percent per shard of ~1% for every 1 million vectors. + The percentage of vectors to explore per shard while doing knn search with `bbq_disk`. Must be between 0 and 100. 0 will default to using `num_candidates` for calculating the percent visited. Increasing `visit_percentage` tends to improve the accuracy of the final results. If `visit_percentage` is set for `bbq_disk`, `num_candidates` is ignored. Defaults to ~1% per shard for every 1 million vectors. `filter` diff --git a/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md b/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md index 3105aea743ca6..36e3ae810921f 100644 --- a/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md +++ b/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md @@ -94,7 +94,7 @@ If all queried fields are of type [semantic_text](/reference/elasticsearch/mappi stack: ga 9.2 ``` `visit_percentage` -: (Optional, float) The percentage of vectors to explore per shard while doing knn search with `bbq_disk`. Must be between 1 and 100. 0 will default to using `num_candidates` for calculating the percent visited. Increasing `visit_percentage` tends to improve the accuracy of the final results. If `visit_percentage` is set for `bbq_disk`, `num_candidates` is ignored. Defaults to 1% per shard for every 1 million vectors. +: (Optional, float) The percentage of vectors to explore per shard while doing knn search with `bbq_disk`. Must be between 0 and 100. 0 will default to using `num_candidates` for calculating the percent visited. Increasing `visit_percentage` tends to improve the accuracy of the final results. If `visit_percentage` is set for `bbq_disk`, `num_candidates` is ignored. Defaults to ~1% per shard for every 1 million vectors. `filter` From 71e0c4330c8cc8d399689728b88b2a9c3c887302 Mon Sep 17 00:00:00 2001 From: John Wagster Date: Tue, 2 Sep 2025 09:44:48 -0500 Subject: [PATCH 26/36] removed unnecessary ctor and made visitPercentage a Float intead of a float in all ctors --- .../search/retriever/KnnRetrieverBuilder.java | 12 ------------ .../search/vectors/KnnSearchBuilder.java | 19 +++++++++---------- .../xpack/rank/linear/LinearRetrieverIT.java | 15 ++++++++------- .../xpack/rank/rrf/RRFRetrieverBuilderIT.java | 18 ++++++++++-------- .../rrf/RRFRetrieverBuilderNestedDocsIT.java | 2 +- 5 files changed, 28 insertions(+), 38 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java index a8700c7d284cb..e2f1583f80db3 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java @@ -118,18 +118,6 @@ public static KnnRetrieverBuilder fromXContent(XContentParser parser, RetrieverP private final RescoreVectorBuilder rescoreVectorBuilder; private final Float similarity; - public KnnRetrieverBuilder( - String field, - float[] queryVector, - QueryVectorBuilder queryVectorBuilder, - int k, - int numCands, - RescoreVectorBuilder rescoreVectorBuilder, - Float similarity - ) { - this(field, queryVector, queryVectorBuilder, k, numCands, null, rescoreVectorBuilder, similarity); - } - public KnnRetrieverBuilder( String field, float[] queryVector, diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index b0940dc946379..ef24c05fd3ba5 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -147,7 +147,7 @@ public KnnSearchBuilder( float[] queryVector, int k, int numCands, - float visitPercentage, + Float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity ) { @@ -177,7 +177,7 @@ public KnnSearchBuilder( VectorData queryVector, int k, int numCands, - float visitPercentage, + Float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity ) { @@ -198,7 +198,7 @@ public KnnSearchBuilder( QueryVectorBuilder queryVectorBuilder, int k, int numCands, - float visitPercentage, + Float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity ) { @@ -220,7 +220,7 @@ public KnnSearchBuilder( QueryVectorBuilder queryVectorBuilder, int k, int numCands, - float visitPercentage, + Float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity ) { @@ -255,7 +255,7 @@ private KnnSearchBuilder( this.queryVectorBuilder = null; this.k = k; this.numCands = numCands; - this.visitPercentage = visitPercentage; + this.visitPercentage = visitPercentage == null ? 0.0f : visitPercentage; this.filterQueries = filterQueries; this.querySupplier = querySupplier; this.similarity = similarity; @@ -287,7 +287,7 @@ private KnnSearchBuilder( if (numCandidates > NUM_CANDS_LIMIT) { throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot exceed [" + NUM_CANDS_LIMIT + "]"); } - if (visitPercentage < 0.0f || visitPercentage > 100.0f) { + if (visitPercentage != null && (visitPercentage < 0.0f || visitPercentage > 100.0f)) { throw new IllegalArgumentException("[" + VISIT_PERCENTAGE_FIELD.getPreferredName() + "] must be between 0 and 100"); } if (queryVector == null && queryVectorBuilder == null) { @@ -313,7 +313,7 @@ private KnnSearchBuilder( this.queryVectorBuilder = queryVectorBuilder; this.k = k; this.numCands = numCandidates; - this.visitPercentage = visitPercentage; + this.visitPercentage = visitPercentage == null ? 0.0f : visitPercentage;; this.rescoreVectorBuilder = rescoreVectorBuilder; this.innerHitBuilder = innerHitBuilder; this.similarity = similarity; @@ -514,7 +514,7 @@ public boolean equals(Object o) { KnnSearchBuilder that = (KnnSearchBuilder) o; return k == that.k && numCands == that.numCands - && visitPercentage == that.visitPercentage + && Objects.equals(visitPercentage, that.visitPercentage) && Objects.equals(rescoreVectorBuilder, that.rescoreVectorBuilder) && Objects.equals(field, that.field) && Objects.equals(queryVector, that.queryVector) @@ -718,7 +718,6 @@ public KnnSearchBuilder build(int size) { int adjustedNumCandidates = numCandidates == null ? Math.round(Math.min(NUM_CANDS_LIMIT, NUM_CANDS_MULTIPLICATIVE_FACTOR * adjustedK)) : numCandidates; - float adjustedVisitPercentage = visitPercentage == null ? 0.0f : visitPercentage; return new KnnSearchBuilder( field, queryVectorBuilder, @@ -726,7 +725,7 @@ public KnnSearchBuilder build(int size) { filterQueries, adjustedK, adjustedNumCandidates, - adjustedVisitPercentage, + visitPercentage, rescoreVectorBuilder, similarity, innerHitBuilder, diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverIT.java index 3d01530d6125f..cb075f473a0b2 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverIT.java @@ -175,7 +175,7 @@ public void testLinearRetrieverWithAggs() { ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null, null); // all requests would have an equal weight and use the identity normalizer source.retriever( @@ -233,7 +233,7 @@ public void testLinearWithCollapse() { standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 // with scores 1, 0.5, 0.05882353, 0.03846154 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null, null); // final ranking with no-normalizer would be: doc 2, 6, 1, 4, 7, 3 // doc 1: 10 // doc 2: 9 + 20 + 1 = 30 @@ -302,7 +302,7 @@ public void testLinearRetrieverWithCollapseAndAggs() { standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 // with scores 1, 0.5, 0.05882353, 0.03846154 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null,null, null); // final ranking with no-normalizer would be: doc 2, 6, 1, 4, 7, 3 // doc 1: 10 // doc 2: 9 + 20 + 1 = 30 @@ -393,7 +393,7 @@ public void testMultipleLinearRetrievers() { ), // this one bring just doc 7 which should be ranked first eventually with a score of 100 new CompoundRetrieverBuilder.RetrieverSource( - new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 7.0f }, null, 1, 100, null, null), + new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 7.0f }, null, 1, 100, null,null, null), null ) ), @@ -447,7 +447,7 @@ public void testLinearExplainWithNamedRetrievers() { standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 // with scores 1, 0.5, 0.05882353, 0.03846154 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null,null, null); // final ranking with no-normalizer would be: doc 2, 6, 1, 4, 7, 3 // doc 1: 10 // doc 2: 9 + 20 + 1 = 30 @@ -537,7 +537,7 @@ public void testLinearExplainWithAnotherNestedLinear() { standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 // with scores 1, 0.5, 0.05882353, 0.03846154 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null,null, null); // final ranking with no-normalizer would be: doc 2, 6, 1, 4, 7, 3 // doc 1: 10 // doc 2: 9 + 20 + 1 = 30 @@ -764,6 +764,7 @@ public void testLinearFiltersPropagatedToKnnQueryVectorBuilder() { 10, 10, null, + null, null ); source.retriever( @@ -816,7 +817,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws throw new IllegalStateException("Should not be called"); } }; - var knn = new KnnRetrieverBuilder("vector", null, vectorBuilder, 10, 10, null, null); + var knn = new KnnRetrieverBuilder("vector", null, vectorBuilder, 10, 10, null,null, null); var standard = new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", vectorBuilder, 10, 10, 10f, null)); var rrf = new LinearRetrieverBuilder( List.of(new CompoundRetrieverBuilder.RetrieverSource(knn, null), new CompoundRetrieverBuilder.RetrieverSource(standard, null)), diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java index ad5a6464f12cd..1514e84619832 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java @@ -190,6 +190,7 @@ public void testRRFPagination() { 10, 100, null, + null, null ); source.retriever( @@ -241,7 +242,7 @@ public void testRRFWithAggs() { ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -296,7 +297,7 @@ public void testRRFWithCollapse() { ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null,null, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -353,7 +354,7 @@ public void testRRFRetrieverWithCollapseAndAggs() { ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null,null, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -419,7 +420,7 @@ public void testMultipleRRFRetrievers() { ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null,null, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -438,7 +439,7 @@ public void testMultipleRRFRetrievers() { ), // this one bring just doc 7 which should be ranked first eventually new CompoundRetrieverBuilder.RetrieverSource( - new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 7.0f }, null, 1, 100, null, null), + new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 7.0f }, null, 1, 100, null,null, null), null ) ), @@ -485,7 +486,7 @@ public void testRRFExplainWithNamedRetrievers() { ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null,null, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -544,7 +545,7 @@ public void testRRFExplainWithAnotherNestedRRF() { ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null,null, null); RRFRetrieverBuilder nestedRRF = new RRFRetrieverBuilder( Arrays.asList( @@ -765,6 +766,7 @@ public void testRRFFiltersPropagatedToKnnQueryVectorBuilder() { 10, 10, null, + null, null ); source.retriever( @@ -818,7 +820,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws throw new IllegalStateException("Should not be called"); } }; - var knn = new KnnRetrieverBuilder("vector", null, vectorBuilder, 10, 10, null, null); + var knn = new KnnRetrieverBuilder("vector", null, vectorBuilder, 10, 10, null,null, null); var standard = new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", vectorBuilder, 10, 10, 10f, null)); var rrf = new RRFRetrieverBuilder( List.of(new CompoundRetrieverBuilder.RetrieverSource(knn, null), new CompoundRetrieverBuilder.RetrieverSource(standard, null)), diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java index a00b940bbed62..29c56bff20ab4 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java @@ -149,7 +149,7 @@ public void testRRFRetrieverWithNestedQuery() { ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 6.0f }, null, 1, 100, null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 6.0f }, null, 1, 100, null,null, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( From bf8c01d8332314875f448960a4b200d248a94688 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 2 Sep 2025 14:52:48 +0000 Subject: [PATCH 27/36] [CI] Auto commit changes from spotless --- .../search/vectors/KnnSearchBuilder.java | 3 +- .../xpack/rank/linear/LinearRetrieverIT.java | 59 ++++++++++++++-- .../xpack/rank/rrf/RRFRetrieverBuilderIT.java | 70 ++++++++++++++++--- .../rrf/RRFRetrieverBuilderNestedDocsIT.java | 11 ++- 4 files changed, 126 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index ef24c05fd3ba5..f5b3ddc934410 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -313,7 +313,8 @@ private KnnSearchBuilder( this.queryVectorBuilder = queryVectorBuilder; this.k = k; this.numCands = numCandidates; - this.visitPercentage = visitPercentage == null ? 0.0f : visitPercentage;; + this.visitPercentage = visitPercentage == null ? 0.0f : visitPercentage; + ; this.rescoreVectorBuilder = rescoreVectorBuilder; this.innerHitBuilder = innerHitBuilder; this.similarity = similarity; diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverIT.java index cb075f473a0b2..86e835c21efe1 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverIT.java @@ -175,7 +175,16 @@ public void testLinearRetrieverWithAggs() { ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 2.0f }, + null, + 10, + 100, + null, + null, + null + ); // all requests would have an equal weight and use the identity normalizer source.retriever( @@ -233,7 +242,16 @@ public void testLinearWithCollapse() { standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 // with scores 1, 0.5, 0.05882353, 0.03846154 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 2.0f }, + null, + 10, + 100, + null, + null, + null + ); // final ranking with no-normalizer would be: doc 2, 6, 1, 4, 7, 3 // doc 1: 10 // doc 2: 9 + 20 + 1 = 30 @@ -302,7 +320,16 @@ public void testLinearRetrieverWithCollapseAndAggs() { standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 // with scores 1, 0.5, 0.05882353, 0.03846154 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null,null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 2.0f }, + null, + 10, + 100, + null, + null, + null + ); // final ranking with no-normalizer would be: doc 2, 6, 1, 4, 7, 3 // doc 1: 10 // doc 2: 9 + 20 + 1 = 30 @@ -393,7 +420,7 @@ public void testMultipleLinearRetrievers() { ), // this one bring just doc 7 which should be ranked first eventually with a score of 100 new CompoundRetrieverBuilder.RetrieverSource( - new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 7.0f }, null, 1, 100, null,null, null), + new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 7.0f }, null, 1, 100, null, null, null), null ) ), @@ -447,7 +474,16 @@ public void testLinearExplainWithNamedRetrievers() { standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 // with scores 1, 0.5, 0.05882353, 0.03846154 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null,null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 2.0f }, + null, + 10, + 100, + null, + null, + null + ); // final ranking with no-normalizer would be: doc 2, 6, 1, 4, 7, 3 // doc 1: 10 // doc 2: 9 + 20 + 1 = 30 @@ -537,7 +573,16 @@ public void testLinearExplainWithAnotherNestedLinear() { standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 // with scores 1, 0.5, 0.05882353, 0.03846154 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null,null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 2.0f }, + null, + 10, + 100, + null, + null, + null + ); // final ranking with no-normalizer would be: doc 2, 6, 1, 4, 7, 3 // doc 1: 10 // doc 2: 9 + 20 + 1 = 30 @@ -817,7 +862,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws throw new IllegalStateException("Should not be called"); } }; - var knn = new KnnRetrieverBuilder("vector", null, vectorBuilder, 10, 10, null,null, null); + var knn = new KnnRetrieverBuilder("vector", null, vectorBuilder, 10, 10, null, null, null); var standard = new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", vectorBuilder, 10, 10, 10f, null)); var rrf = new LinearRetrieverBuilder( List.of(new CompoundRetrieverBuilder.RetrieverSource(knn, null), new CompoundRetrieverBuilder.RetrieverSource(standard, null)), diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java index 1514e84619832..1a5cd6bca3059 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java @@ -242,7 +242,16 @@ public void testRRFWithAggs() { ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null, null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 2.0f }, + null, + 10, + 100, + null, + null, + null + ); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -297,7 +306,16 @@ public void testRRFWithCollapse() { ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null,null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 2.0f }, + null, + 10, + 100, + null, + null, + null + ); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -354,7 +372,16 @@ public void testRRFRetrieverWithCollapseAndAggs() { ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null,null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 2.0f }, + null, + 10, + 100, + null, + null, + null + ); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -420,7 +447,16 @@ public void testMultipleRRFRetrievers() { ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null,null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 2.0f }, + null, + 10, + 100, + null, + null, + null + ); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -439,7 +475,7 @@ public void testMultipleRRFRetrievers() { ), // this one bring just doc 7 which should be ranked first eventually new CompoundRetrieverBuilder.RetrieverSource( - new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 7.0f }, null, 1, 100, null,null, null), + new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 7.0f }, null, 1, 100, null, null, null), null ) ), @@ -486,7 +522,16 @@ public void testRRFExplainWithNamedRetrievers() { ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null,null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 2.0f }, + null, + 10, + 100, + null, + null, + null + ); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -545,7 +590,16 @@ public void testRRFExplainWithAnotherNestedRRF() { ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null,null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 2.0f }, + null, + 10, + 100, + null, + null, + null + ); RRFRetrieverBuilder nestedRRF = new RRFRetrieverBuilder( Arrays.asList( @@ -820,7 +874,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws throw new IllegalStateException("Should not be called"); } }; - var knn = new KnnRetrieverBuilder("vector", null, vectorBuilder, 10, 10, null,null, null); + var knn = new KnnRetrieverBuilder("vector", null, vectorBuilder, 10, 10, null, null, null); var standard = new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", vectorBuilder, 10, 10, 10f, null)); var rrf = new RRFRetrieverBuilder( List.of(new CompoundRetrieverBuilder.RetrieverSource(knn, null), new CompoundRetrieverBuilder.RetrieverSource(standard, null)), diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java index 29c56bff20ab4..21bbc6ab5180f 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java @@ -149,7 +149,16 @@ public void testRRFRetrieverWithNestedQuery() { ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 6.0f }, null, 1, 100, null,null, null); + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 6.0f }, + null, + 1, + 100, + null, + null, + null + ); source.retriever( new RRFRetrieverBuilder( Arrays.asList( From 99f7c65c7305ad5b430cfab15f37df684913522e Mon Sep 17 00:00:00 2001 From: John Wagster Date: Tue, 2 Sep 2025 12:18:10 -0500 Subject: [PATCH 28/36] pass Float down through KnnSearchBuilder as null --- .../search/vectors/KnnSearchBuilder.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index f5b3ddc934410..fb018c5416e1c 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -124,7 +124,7 @@ public static KnnSearchBuilder.Builder fromXContent(XContentParser parser) throw private final Supplier querySupplier; final int k; final int numCands; - final float visitPercentage; + final Float visitPercentage; final Float similarity; final List filterQueries; String queryName; @@ -255,7 +255,7 @@ private KnnSearchBuilder( this.queryVectorBuilder = null; this.k = k; this.numCands = numCands; - this.visitPercentage = visitPercentage == null ? 0.0f : visitPercentage; + this.visitPercentage = visitPercentage; this.filterQueries = filterQueries; this.querySupplier = querySupplier; this.similarity = similarity; @@ -313,8 +313,7 @@ private KnnSearchBuilder( this.queryVectorBuilder = queryVectorBuilder; this.k = k; this.numCands = numCandidates; - this.visitPercentage = visitPercentage == null ? 0.0f : visitPercentage; - ; + this.visitPercentage = visitPercentage; this.rescoreVectorBuilder = rescoreVectorBuilder; this.innerHitBuilder = innerHitBuilder; this.similarity = similarity; @@ -329,9 +328,9 @@ public KnnSearchBuilder(StreamInput in) throws IOException { this.k = in.readVInt(); this.numCands = in.readVInt(); if (in.getTransportVersion().onOrAfter(TransportVersions.VISIT_PERCENTAGE)) { - this.visitPercentage = in.readFloat(); + this.visitPercentage = in.readOptionalFloat(); } else { - this.visitPercentage = 0.0f; + this.visitPercentage = null; } if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { this.queryVector = in.readOptionalWriteable(VectorData::new); @@ -374,7 +373,7 @@ public int getNumCands() { return numCands; } - public float getVisitPercentage() { + public Float getVisitPercentage() { return visitPercentage; } @@ -553,7 +552,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(K_FIELD.getPreferredName(), k); builder.field(NUM_CANDS_FIELD.getPreferredName(), numCands); - if (visitPercentage != 0.0f) { + if (visitPercentage != null && visitPercentage != 0.0f) { builder.field(VISIT_PERCENTAGE_FIELD.getPreferredName(), visitPercentage); } From 674466ceb842df541c38bf9190fde0ed1849338c Mon Sep 17 00:00:00 2001 From: John Wagster Date: Tue, 2 Sep 2025 21:29:55 -0500 Subject: [PATCH 29/36] fix bad serialization --- .../java/org/elasticsearch/search/vectors/KnnSearchBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index fb018c5416e1c..6ab3b4a064bda 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -601,7 +601,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(k); out.writeVInt(numCands); if (out.getTransportVersion().onOrAfter(TransportVersions.VISIT_PERCENTAGE)) { - out.writeFloat(visitPercentage); + out.writeOptionalFloat(visitPercentage); } if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalWriteable(queryVector); From 454982e7304d6c4aaff655fc0df920e022988c4b Mon Sep 17 00:00:00 2001 From: John Wagster Date: Wed, 3 Sep 2025 10:21:35 -0500 Subject: [PATCH 30/36] Delete docs/changelog/133753.yaml --- docs/changelog/133753.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/133753.yaml diff --git a/docs/changelog/133753.yaml b/docs/changelog/133753.yaml deleted file mode 100644 index 3eb4a93b62363..0000000000000 --- a/docs/changelog/133753.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 133753 -summary: KNN Query `visit_percentage` -area: Vector Search -type: feature -issues: [] From 6c24dd5299e35d00dba6b0ef754988c03f4b0fdc Mon Sep 17 00:00:00 2001 From: John Wagster Date: Wed, 3 Sep 2025 10:34:31 -0500 Subject: [PATCH 31/36] fixing nulls --- .../index/mapper/vectors/DenseVectorFieldMapper.java | 6 +++--- .../search/retriever/KnnRetrieverBuilder.java | 8 ++++---- .../elasticsearch/search/vectors/KnnSearchBuilder.java | 2 +- .../search/vectors/KnnSearchRequestParser.java | 8 ++++---- .../search/vectors/KnnVectorQueryBuilder.java | 3 +-- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 70251b3b0898d..21557acebafc9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2530,7 +2530,7 @@ public Query createKnnQuery( VectorData queryVector, int k, int numCands, - float visitPercentage, + Float visitPercentage, Float oversample, Query filter, Float similarityThreshold, @@ -2695,7 +2695,7 @@ private Query createKnnFloatQuery( float[] queryVector, int k, int numCands, - float visitPercentage, + Float visitPercentage, Float queryOversample, Query filter, Float similarityThreshold, @@ -2744,7 +2744,7 @@ private Query createKnnFloatQuery( .build(); } else if (indexOptions instanceof BBQIVFIndexOptions bbqIndexOptions) { float defaultVisitRatio = (float) (bbqIndexOptions.defaultVisitPercentage / 100d); - float visitRatio = visitPercentage == 0.0f ? defaultVisitRatio : (float) (visitPercentage / 100d); + float visitRatio = visitPercentage == null ? defaultVisitRatio : (float) (visitPercentage / 100d); knnQuery = parentFilter != null ? new DiversifyingChildrenIVFKnnFloatVectorQuery( name(), diff --git a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java index e2f1583f80db3..1847470f5c7ec 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java @@ -114,7 +114,7 @@ public static KnnRetrieverBuilder fromXContent(XContentParser parser, RetrieverP private final QueryVectorBuilder queryVectorBuilder; private final int k; private final int numCands; - private final float visitPercentage; + private final Float visitPercentage; private final RescoreVectorBuilder rescoreVectorBuilder; private final Float similarity; @@ -150,7 +150,7 @@ public KnnRetrieverBuilder( this.queryVectorBuilder = queryVectorBuilder; this.k = k; this.numCands = numCands; - this.visitPercentage = visitPercentage == null ? 0.0f : visitPercentage; + this.visitPercentage = visitPercentage; this.similarity = similarity; this.rescoreVectorBuilder = rescoreVectorBuilder; } @@ -273,7 +273,7 @@ public void doToXContent(XContentBuilder builder, Params params) throws IOExcept builder.field(K_FIELD.getPreferredName(), k); builder.field(NUM_CANDS_FIELD.getPreferredName(), numCands); - if (visitPercentage != 0.0f) { + if (visitPercentage != null) { builder.field(VISIT_PERCENTAGE_FIELD.getPreferredName(), visitPercentage); } @@ -299,7 +299,7 @@ public boolean doEquals(Object o) { KnnRetrieverBuilder that = (KnnRetrieverBuilder) o; return k == that.k && numCands == that.numCands - && visitPercentage == that.visitPercentage + && Objects.equals(visitPercentage, that.visitPercentage) && Objects.equals(field, that.field) && ((queryVector == null && that.queryVector == null) || (queryVector != null && that.queryVector != null && Arrays.equals(queryVector.get(), that.queryVector.get()))) diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index 6ab3b4a064bda..1091f846a5d06 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -552,7 +552,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(K_FIELD.getPreferredName(), k); builder.field(NUM_CANDS_FIELD.getPreferredName(), numCands); - if (visitPercentage != null && visitPercentage != 0.0f) { + if (visitPercentage != null) { builder.field(VISIT_PERCENTAGE_FIELD.getPreferredName(), visitPercentage); } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java index 585f12b36b8bf..c4ff1ca40c5ab 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java @@ -232,7 +232,7 @@ public static KnnSearch parse(XContentParser parser) throws IOException { final float[] queryVector; final int k; final int numCands; - final float visitPercentage; + final Float visitPercentage; /** * Defines a kNN search. @@ -247,7 +247,7 @@ public static KnnSearch parse(XContentParser parser) throws IOException { this.queryVector = queryVector; this.k = k; this.numCands = numCands; - this.visitPercentage = visitPercentage == null ? 0.0f : visitPercentage; + this.visitPercentage = visitPercentage; } public KnnVectorQueryBuilder toQueryBuilder() { @@ -264,7 +264,7 @@ public KnnVectorQueryBuilder toQueryBuilder() { if (numCands > NUM_CANDS_LIMIT) { throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot exceed [" + NUM_CANDS_LIMIT + "]"); } - if (visitPercentage < 0.0f || visitPercentage > 100.0f) { + if (visitPercentage != null && (visitPercentage < 0.0f || visitPercentage > 100.0f)) { throw new IllegalArgumentException("[" + VISIT_PERCENTAGE_FIELD.getPreferredName() + "] must be between 0 and 100"); } return new KnnVectorQueryBuilder(field, queryVector, numCands, numCands, visitPercentage, null, null); @@ -277,7 +277,7 @@ public boolean equals(Object o) { KnnSearch that = (KnnSearch) o; return k == that.k && numCands == that.numCands - && visitPercentage == that.visitPercentage + && Objects.equals(visitPercentage, that.visitPercentage) && Objects.equals(field, that.field) && Arrays.equals(queryVector, that.queryVector); } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index 7bd3edc2e62c5..700ed4748dc94 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -570,7 +570,6 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { } } int adjustedNumCands = numCands == null ? Math.round(Math.min(NUM_CANDS_MULTIPLICATIVE_FACTOR * k, NUM_CANDS_LIMIT)) : numCands; - float adjustedVisitPercentage = visitPercentage == null ? 0.0f : visitPercentage; if (fieldType == null) { return new MatchNoDocsQuery(); @@ -635,7 +634,7 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { queryVector, k, adjustedNumCands, - adjustedVisitPercentage, + visitPercentage, oversample, filterQuery, vectorSimilarity, From 3f20296b7b6013bdc5cd18ec91ec39c21c343fa3 Mon Sep 17 00:00:00 2001 From: John Wagster Date: Wed, 3 Sep 2025 14:04:12 -0500 Subject: [PATCH 32/36] improving tests with null checks and added diskbbq format to densevectorfieldtype test --- .../search/query/RescoreKnnVectorQueryIT.java | 4 +-- .../vectors/DenseVectorFieldTypeTests.java | 32 ++++++++++++++++--- .../KnnRetrieverBuilderParsingTests.java | 2 +- .../RankDocsRetrieverBuilderTests.java | 2 +- ...AbstractKnnVectorQueryBuilderTestCase.java | 2 +- .../search/vectors/KnnSearchBuilderTests.java | 4 +-- .../vectors/KnnSearchRequestParserTests.java | 2 +- .../search/RandomSearchRequestGenerator.java | 2 +- 8 files changed, 36 insertions(+), 14 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java index 875cbbe4de484..efffcb6951ae2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java @@ -116,7 +116,7 @@ private record TestParams( float[] queryVector, int k, int numCands, - float visitPercentage, + Float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder ) { public static TestParams generate() { @@ -129,7 +129,7 @@ public static TestParams generate() { randomVector(numDims), k, (int) (k * randomFloatBetween(1.0f, 10.0f, true)), - randomFloatBetween(0.0f, 100.0f, true), + randomBoolean() ? null : randomFloatBetween(0.0f, 100.0f, true), new RescoreVectorBuilder(randomFloatBetween(1.0f, 100f, true)) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index beb9e7dd828bd..013966bb9e561 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -25,22 +25,29 @@ import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.VectorSimilarity; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.vectors.DenseVectorQuery; +import org.elasticsearch.search.vectors.DiversifyingChildrenIVFKnnFloatVectorQuery; import org.elasticsearch.search.vectors.DiversifyingParentBlockQuery; import org.elasticsearch.search.vectors.ESKnnByteVectorQuery; import org.elasticsearch.search.vectors.ESKnnFloatVectorQuery; +import org.elasticsearch.search.vectors.IVFKnnFloatVectorQuery; import org.elasticsearch.search.vectors.RescoreKnnVectorQuery; import org.elasticsearch.search.vectors.VectorData; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Set; import java.util.function.Function; +import static org.elasticsearch.index.codec.vectors.IVFVectorsFormat.MAX_VECTORS_PER_CLUSTER; +import static org.elasticsearch.index.codec.vectors.IVFVectorsFormat.MIN_VECTORS_PER_CLUSTER; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.BBQ_MIN_DIMS; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType.BIT; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType.BYTE; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType.FLOAT; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.IVF_FORMAT; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.OVERSAMPLE_LIMIT; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -66,7 +73,7 @@ private DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsNonQuan } public static DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsAll() { - return randomFrom( + List options = new ArrayList<>(Arrays.asList( new DenseVectorFieldMapper.HnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000)), new DenseVectorFieldMapper.Int8HnswIndexOptions( randomIntBetween(1, 100), @@ -95,7 +102,18 @@ public static DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsA randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) ), new DenseVectorFieldMapper.BBQFlatIndexOptions(randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())) - ); + )); + + if(IVF_FORMAT.isEnabled()) { + options.add( + new DenseVectorFieldMapper.BBQIVFIndexOptions( + randomIntBetween(MIN_VECTORS_PER_CLUSTER, MAX_VECTORS_PER_CLUSTER), + randomFloatBetween(0.0f, 100.0f, true), + randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())) + ); + } + + return randomFrom(options); } private DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsHnswQuantized() { @@ -244,7 +262,9 @@ public void testCreateNestedKnnQuery() { if (field.getIndexOptions().isFlat()) { assertThat(query, instanceOf(DiversifyingParentBlockQuery.class)); } else { - assertTrue(query instanceof DiversifyingChildrenFloatKnnVectorQuery || query instanceof PatienceKnnVectorQuery); + assertTrue(query instanceof DiversifyingChildrenFloatKnnVectorQuery + || query instanceof PatienceKnnVectorQuery + || query instanceof DiversifyingChildrenIVFKnnFloatVectorQuery); } } { @@ -475,7 +495,9 @@ public void testCreateKnnQueryMaxDims() { if (fieldWith4096dims.getIndexOptions().isFlat()) { assertThat(query, instanceOf(DenseVectorQuery.Floats.class)); } else { - assertTrue(query instanceof KnnFloatVectorQuery || query instanceof PatienceKnnVectorQuery); + assertTrue(query instanceof KnnFloatVectorQuery + || query instanceof PatienceKnnVectorQuery + || query instanceof IVFKnnFloatVectorQuery); } } @@ -792,7 +814,7 @@ private static void checkRescoreQueryParameters( DenseVectorFieldType fieldType, int k, int candidates, - float visitPercentage, + Float visitPercentage, float oversample, int expectedK, int expectedCandidates, diff --git a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java index 4f45b2c9a473d..efe08cc62ab6e 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java @@ -52,7 +52,7 @@ public static KnnRetrieverBuilder createRandomKnnRetrieverBuilder() { float[] vector = randomVector(dim); int k = randomIntBetween(1, 100); int numCands = randomIntBetween(k + 20, 1000); - float visitPercentage = randomFloatBetween(0.0f, 100.0f, true); + Float visitPercentage = randomBoolean() ? null : randomFloatBetween(0.0f, 100.0f, true); Float similarity = randomBoolean() ? null : randomFloat(); RescoreVectorBuilder rescoreVectorBuilder = randomBoolean() ? null diff --git a/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java b/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java index 94aa7c2761470..772088296f76f 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java @@ -70,7 +70,7 @@ private List innerRetrievers(QueryRewriteContext queryRewriteC null, randomInt(10), randomIntBetween(10, 100), - randomFloatBetween(0.0f, 100.0f, true), + randomBoolean() ? null : randomFloatBetween(0.0f, 100.0f, true), randomBoolean() ? null : new RescoreVectorBuilder(randomFloatBetween(1.0f, 10.0f, false)), randomFloat() ); diff --git a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java index 2d0a9aaf6e641..5faeef27c968f 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java @@ -146,7 +146,7 @@ protected KnnVectorQueryBuilder doCreateTestQueryBuilder() { String fieldName = randomBoolean() ? VECTOR_FIELD : VECTOR_ALIAS_FIELD; int k = randomIntBetween(1, 100); int numCands = randomIntBetween(k + 20, 1000); - float visitPercentage = randomFloatBetween(0.0f, 100.0f, true); + Float visitPercentage = randomBoolean() ? null : randomFloatBetween(0.0f, 100.0f, true); KnnVectorQueryBuilder queryBuilder = createKnnVectorQueryBuilder( fieldName, k, diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java index c2555b9112db8..328b295f1f220 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java @@ -52,7 +52,7 @@ public static KnnSearchBuilder randomTestInstance() { float[] vector = randomVector(dim); int k = randomIntBetween(1, 100); int numCands = randomIntBetween(k + 20, 1000); - float visitPercentage = randomFloatBetween(0.0f, 100.0f, true); + Float visitPercentage = randomBoolean() ? null : randomFloatBetween(0.0f, 100.0f, true); RescoreVectorBuilder rescoreVectorBuilder = randomBoolean() ? null : new RescoreVectorBuilder(randomFloatBetween(1.0f, 10.0f, false)); @@ -230,7 +230,7 @@ public void testToQueryBuilder() { float[] vector = randomVector(randomIntBetween(2, 30)); int k = randomIntBetween(1, 100); int numCands = randomIntBetween(k, 1000); - float visitPercentage = randomFloatBetween(0.0f, 100.0f, true); + Float visitPercentage = randomBoolean() ? null : randomFloatBetween(0.0f, 100.0f, true); Float similarity = randomBoolean() ? null : randomFloat(); RescoreVectorBuilder rescoreVectorBuilder = randomBoolean() ? null diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java index 3649cea2260eb..80c00bdf4907c 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java @@ -277,7 +277,7 @@ private KnnSearch randomKnnSearch() { int k = randomIntBetween(1, 100); int numCands = randomIntBetween(k, 1000); - float visitPercentage = randomFloatBetween(0.0f, 100.0f, true); + Float visitPercentage = randomBoolean() ? null : randomFloatBetween(0.0f, 100.0f, true); return new KnnSearch(field, vector, k, numCands, visitPercentage); } diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index 5f91b76387211..c8a2fa10e33af 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -266,7 +266,7 @@ public static SearchSourceBuilder randomSearchSourceBuilder( } int k = randomIntBetween(1, 100); int numCands = randomIntBetween(k, 1000); - float visitPercentage = randomFloatBetween(0.0f, 100.0f, true); + Float visitPercentage = randomBoolean() ? null : randomFloatBetween(0.0f, 100.0f, true); RescoreVectorBuilder rescoreVectorBuilder = randomBoolean() ? null : new RescoreVectorBuilder(randomFloatBetween(1.0f, 10.0f, false)); From 84aeeeb496572c637440746a89c2c3020272df5d Mon Sep 17 00:00:00 2001 From: John Wagster Date: Wed, 3 Sep 2025 14:05:00 -0500 Subject: [PATCH 33/36] spotless --- .../vectors/DenseVectorFieldTypeTests.java | 85 ++++++++++--------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 013966bb9e561..d6206b845ae25 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -73,43 +73,48 @@ private DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsNonQuan } public static DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsAll() { - List options = new ArrayList<>(Arrays.asList( - new DenseVectorFieldMapper.HnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000)), - new DenseVectorFieldMapper.Int8HnswIndexOptions( - randomIntBetween(1, 100), - randomIntBetween(1, 10_000), - randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), - randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) - ), - new DenseVectorFieldMapper.Int4HnswIndexOptions( - randomIntBetween(1, 100), - randomIntBetween(1, 10_000), - randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), - randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) - ), - new DenseVectorFieldMapper.FlatIndexOptions(), - new DenseVectorFieldMapper.Int8FlatIndexOptions( - randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), - randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) - ), - new DenseVectorFieldMapper.Int4FlatIndexOptions( - randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), - randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) - ), - new DenseVectorFieldMapper.BBQHnswIndexOptions( - randomIntBetween(1, 100), - randomIntBetween(1, 10_000), - randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) - ), - new DenseVectorFieldMapper.BBQFlatIndexOptions(randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())) - )); + List options = new ArrayList<>( + Arrays.asList( + new DenseVectorFieldMapper.HnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000)), + new DenseVectorFieldMapper.Int8HnswIndexOptions( + randomIntBetween(1, 100), + randomIntBetween(1, 10_000), + randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), + randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) + ), + new DenseVectorFieldMapper.Int4HnswIndexOptions( + randomIntBetween(1, 100), + randomIntBetween(1, 10_000), + randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), + randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) + ), + new DenseVectorFieldMapper.FlatIndexOptions(), + new DenseVectorFieldMapper.Int8FlatIndexOptions( + randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), + randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) + ), + new DenseVectorFieldMapper.Int4FlatIndexOptions( + randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), + randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) + ), + new DenseVectorFieldMapper.BBQHnswIndexOptions( + randomIntBetween(1, 100), + randomIntBetween(1, 10_000), + randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) + ), + new DenseVectorFieldMapper.BBQFlatIndexOptions( + randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) + ) + ) + ); - if(IVF_FORMAT.isEnabled()) { + if (IVF_FORMAT.isEnabled()) { options.add( new DenseVectorFieldMapper.BBQIVFIndexOptions( randomIntBetween(MIN_VECTORS_PER_CLUSTER, MAX_VECTORS_PER_CLUSTER), randomFloatBetween(0.0f, 100.0f, true), - randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())) + randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) + ) ); } @@ -262,9 +267,11 @@ public void testCreateNestedKnnQuery() { if (field.getIndexOptions().isFlat()) { assertThat(query, instanceOf(DiversifyingParentBlockQuery.class)); } else { - assertTrue(query instanceof DiversifyingChildrenFloatKnnVectorQuery - || query instanceof PatienceKnnVectorQuery - || query instanceof DiversifyingChildrenIVFKnnFloatVectorQuery); + assertTrue( + query instanceof DiversifyingChildrenFloatKnnVectorQuery + || query instanceof PatienceKnnVectorQuery + || query instanceof DiversifyingChildrenIVFKnnFloatVectorQuery + ); } } { @@ -495,9 +502,11 @@ public void testCreateKnnQueryMaxDims() { if (fieldWith4096dims.getIndexOptions().isFlat()) { assertThat(query, instanceOf(DenseVectorQuery.Floats.class)); } else { - assertTrue(query instanceof KnnFloatVectorQuery - || query instanceof PatienceKnnVectorQuery - || query instanceof IVFKnnFloatVectorQuery); + assertTrue( + query instanceof KnnFloatVectorQuery + || query instanceof PatienceKnnVectorQuery + || query instanceof IVFKnnFloatVectorQuery + ); } } From 52ae45691505bbd8c606ee83adf1bc4810df2f00 Mon Sep 17 00:00:00 2001 From: John Wagster Date: Wed, 3 Sep 2025 14:49:42 -0500 Subject: [PATCH 34/36] removed docs changes to a separate PR --- .../mapping-reference/dense-vector.md | 17 +++++------------ .../rest-apis/retrievers/knn-retriever.md | 9 --------- .../query-dsl/query-dsl-knn-query.md | 14 ++------------ 3 files changed, 7 insertions(+), 33 deletions(-) diff --git a/docs/reference/elasticsearch/mapping-reference/dense-vector.md b/docs/reference/elasticsearch/mapping-reference/dense-vector.md index db93d9221296c..de6d82104c005 100644 --- a/docs/reference/elasticsearch/mapping-reference/dense-vector.md +++ b/docs/reference/elasticsearch/mapping-reference/dense-vector.md @@ -341,19 +341,18 @@ $$$dense-vector-index-options$$$ `type` : (Required, string) The type of kNN algorithm to use. Can be either any of: * `hnsw` - This utilizes the [HNSW algorithm](https://arxiv.org/abs/1603.09320) for scalable approximate kNN search. This supports all `element_type` values. - * `int8_hnsw` - The default index type for some float vectors: + * `int8_hnsw` - The default index type for some float vectors: * {applies_to}`stack: ga 9.1` Default for float vectors with less than 384 dimensions. * {applies_to}`stack: ga 9.0` Default for float all vectors. This utilizes the [HNSW algorithm](https://arxiv.org/abs/1603.09320) in addition to automatically scalar quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint by 4x at the cost of some accuracy. See [Automatically quantize vectors for kNN search](#dense-vector-quantization). * `int4_hnsw` - This utilizes the [HNSW algorithm](https://arxiv.org/abs/1603.09320) in addition to automatically scalar quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint by 8x at the cost of some accuracy. See [Automatically quantize vectors for kNN search](#dense-vector-quantization). * `bbq_hnsw` - This utilizes the [HNSW algorithm](https://arxiv.org/abs/1603.09320) in addition to automatically binary quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint by 32x at the cost of accuracy. See [Automatically quantize vectors for kNN search](#dense-vector-quantization). - + {applies_to}`stack: ga 9.1` `bbq_hnsw` is the default index type for float vectors with greater than or equal to 384 dimensions. * `flat` - This utilizes a brute-force search algorithm for exact kNN search. This supports all `element_type` values. - * `int8_flat` - This utilizes a brute-force search algorithm in addition to automatic scalar quantization. Only supports `element_type` of `float`. - * `int4_flat` - This utilizes a brute-force search algorithm in addition to automatic half-byte scalar quantization. Only supports `element_type` of `float`. - * `bbq_flat` - This utilizes a brute-force search algorithm in addition to automatic binary quantization. Only supports `element_type` of `float`. - * {applies_to}`stack: ga 9.2` `bbq_disk` - This utilizes a clustering search algorithm in addition to automatic binary quantization. Only supports `element_type` of `float`. + * `int8_flat` - This utilizes a brute-force search algorithm in addition to automatically scalar quantization. Only supports `element_type` of `float`. + * `int4_flat` - This utilizes a brute-force search algorithm in addition to automatically half-byte scalar quantization. Only supports `element_type` of `float`. + * `bbq_flat` - This utilizes a brute-force search algorithm in addition to automatically binary quantization. Only supports `element_type` of `float`. `m` : (Optional, integer) The number of neighbors each node will be connected to in the HNSW graph. Defaults to `16`. Only applicable to `hnsw`, `int8_hnsw`, `int4_hnsw` and `bbq_hnsw` index types. @@ -364,12 +363,6 @@ $$$dense-vector-index-options$$$ `confidence_interval` : (Optional, float) Only applicable to `int8_hnsw`, `int4_hnsw`, `int8_flat`, and `int4_flat` index types. The confidence interval to use when quantizing the vectors. Can be any value between and including `0.90` and `1.0` or exactly `0`. When the value is `0`, this indicates that dynamic quantiles should be calculated for optimized quantization. When between `0.90` and `1.0`, this value restricts the values used when calculating the quantization thresholds. For example, a value of `0.95` will only use the middle 95% of the values when calculating the quantization thresholds (e.g. the highest and lowest 2.5% of values will be ignored). Defaults to `1/(dims + 1)` for `int8` quantized vectors and `0` for `int4` for dynamic quantile calculation. -`default_visit_percentage` {applies_to}`stack: ga 9.2` -: (Optional, integer) Only applicable to `bbq_disk`. Must be between 0 and 100. 0 will default to using `num_candidates` for calculating the percent visited. Increasing `default_visit_percentage` tends to improve the accuracy of the final results. Defaults to ~1% per shard for every 1 million vectors. - -`cluster_size` {applies_to}`stack: ga 9.2` -: (Optional, integer) Only applicable to `bbq_disk`. The number of vectors per cluster. Smaller cluster sizes increases accuracy at the cost of performance. Defaults to `384`. Must be a value between `64` and `65536`. - `rescore_vector` {applies_to}`stack: preview 9.0, ga 9.1` : (Optional, object) An optional section that configures automatic vector rescoring on knn queries for the given field. Only applicable to quantized index types. :::::{dropdown} Properties of rescore_vector diff --git a/docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md b/docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md index 2b7f162255035..12da522214383 100644 --- a/docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md +++ b/docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md @@ -41,15 +41,6 @@ A kNN retriever returns top documents from a [k-nearest neighbor search (kNN)](d The number of nearest neighbor candidates to consider per shard. Needs to be greater than `k`, or `size` if `k` is omitted, and cannot exceed 10,000. {{es}} collects `num_candidates` results from each shard, then merges them to find the top `k` results. Increasing `num_candidates` tends to improve the accuracy of the final `k` results. Defaults to `Math.min(1.5 * k, 10_000)`. -```{applies_to} -stack: ga 9.2 -``` -`visit_percentage` -: (Optional, float) - - The percentage of vectors to explore per shard while doing knn search with `bbq_disk`. Must be between 0 and 100. 0 will default to using `num_candidates` for calculating the percent visited. Increasing `visit_percentage` tends to improve the accuracy of the final results. If `visit_percentage` is set for `bbq_disk`, `num_candidates` is ignored. Defaults to ~1% per shard for every 1 million vectors. - - `filter` : (Optional, [query object or list of query objects](/reference/query-languages/querydsl.md)) diff --git a/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md b/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md index 36e3ae810921f..4af581e466a7a 100644 --- a/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md +++ b/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md @@ -2,9 +2,6 @@ navigation_title: "Knn" mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-knn-query.html -applies_to: - stack: all - serverless: all --- # Knn query [query-dsl-knn-query] @@ -90,13 +87,6 @@ If all queried fields are of type [semantic_text](/reference/elasticsearch/mappi : (Optional, integer) The number of nearest neighbor candidates to consider per shard while doing knn search. Cannot exceed 10,000. Increasing `num_candidates` tends to improve the accuracy of the final results. Defaults to `1.5 * k` if `k` is set, or `1.5 * size` if `k` is not set. -```{applies_to} -stack: ga 9.2 -``` -`visit_percentage` -: (Optional, float) The percentage of vectors to explore per shard while doing knn search with `bbq_disk`. Must be between 0 and 100. 0 will default to using `num_candidates` for calculating the percent visited. Increasing `visit_percentage` tends to improve the accuracy of the final results. If `visit_percentage` is set for `bbq_disk`, `num_candidates` is ignored. Defaults to ~1% per shard for every 1 million vectors. - - `filter` : (Optional, query object) Query to filter the documents that can match. The kNN search will return the top documents that also match this filter. The value can be a single query or a list of queries. If `filter` is not provided, all documents are allowed to match. @@ -118,7 +108,7 @@ The filter is a pre-filter, meaning that it is applied **during** the approximat : (Optional, object) Apply oversampling and rescoring to quantized vectors. **Parameters for `rescore_vector`**: - + `oversample` : (Required, float) @@ -126,7 +116,7 @@ The filter is a pre-filter, meaning that it is applied **during** the approximat * Retrieve `num_candidates` candidates per shard. * From these candidates, the top `k * oversample` candidates per shard will be rescored using the original vectors. - * The top `k` rescored candidates will be returned. Must be one of the following values: + * The top `k` rescored candidates will be returned. Must be one of the following values: * \>= 1f to indicate the oversample factor * Exactly `0` to indicate that no oversampling and rescoring should occur. {applies_to}`stack: ga 9.1` From cd47e8cce62358b556b2812e12c09be8d2034c9f Mon Sep 17 00:00:00 2001 From: John Wagster Date: Wed, 3 Sep 2025 15:32:10 -0500 Subject: [PATCH 35/36] missed a couple Floats --- .../search/vectors/AbstractKnnVectorQueryBuilderTestCase.java | 2 +- .../search/vectors/KnnByteVectorQueryBuilderTests.java | 2 +- .../search/vectors/KnnFloatVectorQueryBuilderTests.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java index 5faeef27c968f..c6725e9710f78 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java @@ -89,7 +89,7 @@ abstract KnnVectorQueryBuilder createKnnVectorQueryBuilder( String fieldName, int k, int numCands, - float visitPercentage, + Float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity ); diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java index 86bfde01c9faa..b473a83ffd3eb 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java @@ -22,7 +22,7 @@ protected KnnVectorQueryBuilder createKnnVectorQueryBuilder( String fieldName, int k, int numCands, - float visitPercentage, + Float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity ) { diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnFloatVectorQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnFloatVectorQueryBuilderTests.java index 13f9a8e441e0a..4544cb260e38c 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnFloatVectorQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnFloatVectorQueryBuilderTests.java @@ -22,7 +22,7 @@ KnnVectorQueryBuilder createKnnVectorQueryBuilder( String fieldName, int k, int numCands, - float visitPercentage, + Float visitPercentage, RescoreVectorBuilder rescoreVectorBuilder, Float similarity ) { From 16c9dbc88daaa5c0a9b5b39f14e79ed2dbbbde98 Mon Sep 17 00:00:00 2001 From: John Wagster Date: Wed, 3 Sep 2025 23:16:25 -0500 Subject: [PATCH 36/36] properly gate the parser ctors with the feature flag, and fix tests --- .../search/retriever/KnnRetrieverBuilder.java | 33 ++++++++++----- .../search/vectors/KnnSearchBuilder.java | 26 ++++++++---- .../vectors/KnnSearchRequestParser.java | 6 ++- .../search/vectors/KnnVectorQueryBuilder.java | 41 ++++++++++++------- .../vectors/KnnSearchRequestParserTests.java | 30 ++++++++------ 5 files changed, 91 insertions(+), 45 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java index 1847470f5c7ec..862299c5cae1e 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java @@ -69,16 +69,29 @@ public final class KnnRetrieverBuilder extends RetrieverBuilder { } else { vectorArray = null; } - return new KnnRetrieverBuilder( - (String) args[0], - vectorArray, - (QueryVectorBuilder) args[2], - (int) args[3], - (int) args[4], - (Float) args[5], - (RescoreVectorBuilder) args[7], - (Float) args[6] - ); + if (IVF_FORMAT.isEnabled()) { + return new KnnRetrieverBuilder( + (String) args[0], + vectorArray, + (QueryVectorBuilder) args[2], + (int) args[3], + (int) args[4], + (Float) args[5], + (RescoreVectorBuilder) args[7], + (Float) args[6] + ); + } else { + return new KnnRetrieverBuilder( + (String) args[0], + vectorArray, + (QueryVectorBuilder) args[2], + (int) args[3], + (int) args[4], + null, + (RescoreVectorBuilder) args[6], + (Float) args[5] + ); + } } ); diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index 1091f846a5d06..f455797e68d12 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -63,14 +63,24 @@ public class KnnSearchBuilder implements Writeable, ToXContentFragment, Rewritea @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("knn", args -> { // TODO optimize parsing for when BYTE values are provided - return new Builder().field((String) args[0]) - .queryVector((VectorData) args[1]) - .queryVectorBuilder((QueryVectorBuilder) args[5]) - .k((Integer) args[2]) - .numCandidates((Integer) args[3]) - .visitPercentage((Float) args[4]) - .similarity((Float) args[6]) - .rescoreVectorBuilder((RescoreVectorBuilder) args[7]); + if (IVF_FORMAT.isEnabled()) { + return new Builder().field((String) args[0]) + .queryVector((VectorData) args[1]) + .queryVectorBuilder((QueryVectorBuilder) args[5]) + .k((Integer) args[2]) + .numCandidates((Integer) args[3]) + .visitPercentage((Float) args[4]) + .similarity((Float) args[6]) + .rescoreVectorBuilder((RescoreVectorBuilder) args[7]); + } else { + return new Builder().field((String) args[0]) + .queryVector((VectorData) args[1]) + .queryVectorBuilder((QueryVectorBuilder) args[4]) + .k((Integer) args[2]) + .numCandidates((Integer) args[3]) + .similarity((Float) args[5]) + .rescoreVectorBuilder((RescoreVectorBuilder) args[6]); + } }); static { diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java index c4ff1ca40c5ab..609b9df8c3fed 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java @@ -211,7 +211,11 @@ static class KnnSearch { for (int i = 0; i < vector.size(); i++) { vectorArray[i] = vector.get(i); } - return new KnnSearch((String) args[0], vectorArray, (int) args[2], (int) args[3], (Float) args[4]); + if (IVF_FORMAT.isEnabled()) { + return new KnnSearch((String) args[0], vectorArray, (int) args[2], (int) args[3], (Float) args[4]); + } else { + return new KnnSearch((String) args[0], vectorArray, (int) args[2], (int) args[3], null); + } }); static { diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index 700ed4748dc94..71c452858e1ae 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -72,20 +72,33 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder PARSER = new ConstructingObjectParser<>( - "knn", - args -> new KnnVectorQueryBuilder( - (String) args[0], - (VectorData) args[1], - (QueryVectorBuilder) args[6], - null, - (Integer) args[2], - (Integer) args[3], - (Float) args[4], - (RescoreVectorBuilder) args[7], - (Float) args[5] - ) - ); + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("knn", args -> { + if (IVF_FORMAT.isEnabled()) { + return new KnnVectorQueryBuilder( + (String) args[0], + (VectorData) args[1], + (QueryVectorBuilder) args[6], + null, + (Integer) args[2], + (Integer) args[3], + (Float) args[4], + (RescoreVectorBuilder) args[7], + (Float) args[5] + ); + } else { + return new KnnVectorQueryBuilder( + (String) args[0], + (VectorData) args[1], + (QueryVectorBuilder) args[5], + null, + (Integer) args[2], + (Integer) args[3], + null, + (RescoreVectorBuilder) args[6], + (Float) args[4] + ); + } + }); static { PARSER.declareString(constructorArg(), FIELD_FIELD); diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java index 80c00bdf4907c..38b5bd8b4b475 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java @@ -110,10 +110,12 @@ public void testParseSourceString() throws IOException { .startObject(KnnSearchRequestParser.KNN_SECTION_FIELD.getPreferredName()) .field(KnnSearch.FIELD_FIELD.getPreferredName(), knnSearch.field) .field(KnnSearch.K_FIELD.getPreferredName(), knnSearch.k) - .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), knnSearch.numCands) - .field(KnnSearch.VISIT_PERCENTAGE_FIELD.getPreferredName(), knnSearch.visitPercentage) - .field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), knnSearch.queryVector) - .endObject(); + .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), knnSearch.numCands); + if (knnSearch.visitPercentage != null) { + builder.field(KnnSearch.VISIT_PERCENTAGE_FIELD.getPreferredName(), knnSearch.visitPercentage); + } + builder.field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), knnSearch.queryVector); + builder.endObject(); builder.field(SearchSourceBuilder._SOURCE_FIELD.getPreferredName(), "some-field"); builder.endObject(); @@ -137,10 +139,12 @@ public void testParseSourceArray() throws IOException { .startObject(KnnSearchRequestParser.KNN_SECTION_FIELD.getPreferredName()) .field(KnnSearch.FIELD_FIELD.getPreferredName(), knnSearch.field) .field(KnnSearch.K_FIELD.getPreferredName(), knnSearch.k) - .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), knnSearch.numCands) - .field(KnnSearch.VISIT_PERCENTAGE_FIELD.getPreferredName(), knnSearch.visitPercentage) - .field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), knnSearch.queryVector) - .endObject(); + .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), knnSearch.numCands); + if (knnSearch.visitPercentage != null) { + builder.field(KnnSearch.VISIT_PERCENTAGE_FIELD.getPreferredName(), knnSearch.visitPercentage); + } + builder.field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), knnSearch.queryVector); + builder.endObject(); builder.array(SearchSourceBuilder._SOURCE_FIELD.getPreferredName(), "field1", "field2", "field3"); builder.endObject(); @@ -300,10 +304,12 @@ private XContentBuilder createRequestBody(KnnSearch knnSearch, List