diff --git a/docs/changelog/126401.yaml b/docs/changelog/126401.yaml new file mode 100644 index 0000000000000..227a6c5a61795 --- /dev/null +++ b/docs/changelog/126401.yaml @@ -0,0 +1,5 @@ +pr: 126401 +summary: Add pinned retriever +area: Relevance +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index e363e4e140574..d2d9fe3fea8a4 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -231,6 +231,7 @@ static TransportVersion def(int id) { public static final TransportVersion INTRODUCE_FAILURES_LIFECYCLE = def(9_065_0_00); public static final TransportVersion PROJECT_METADATA_SETTINGS = def(9_066_00_0); public static final TransportVersion AGGREGATE_METRIC_DOUBLE_BLOCK = def(9_067_00_0); + public static final TransportVersion PINNED_RETRIEVER = def(9_068_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/ent-search/build.gradle b/x-pack/plugin/ent-search/build.gradle index bca7547f0cd8c..b9802566b6c2f 100644 --- a/x-pack/plugin/ent-search/build.gradle +++ b/x-pack/plugin/ent-search/build.gradle @@ -8,7 +8,7 @@ esplugin { name = 'x-pack-ent-search' description = 'Elasticsearch Expanded Pack Plugin - Enterprise Search' classname = 'org.elasticsearch.xpack.application.EnterpriseSearch' - extendedPlugins = ['x-pack-core'] + extendedPlugins = ['x-pack-core', 'search-business-rules'] } base { @@ -17,7 +17,7 @@ base { dependencies { compileOnly project(path: xpackModule('core')) - implementation project(xpackModule('search-business-rules')) + compileOnly project(xpackModule('search-business-rules')) api project(':modules:lang-mustache') // JSON Schema dependencies diff --git a/x-pack/plugin/rank-rrf/build.gradle b/x-pack/plugin/rank-rrf/build.gradle index 216e85f48f56f..fa598c6ef677a 100644 --- a/x-pack/plugin/rank-rrf/build.gradle +++ b/x-pack/plugin/rank-rrf/build.gradle @@ -26,6 +26,7 @@ dependencies { clusterModules project(xpackModule('rank-rrf')) clusterModules project(xpackModule('inference')) clusterModules project(':modules:lang-painless') + clusterModules project(xpackModule('search-business-rules')) clusterPlugins project(':x-pack:plugin:inference:qa:test-service-plugin') } diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/950_pinned_interaction.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/950_pinned_interaction.yml new file mode 100644 index 0000000000000..e5629b7715994 --- /dev/null +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/950_pinned_interaction.yml @@ -0,0 +1,98 @@ +setup: + - requires: + cluster_features: 'pinned_retriever_supported' + reason: 'test requires pinned retriever implementation' + - do: + indices.create: + index: test-index1 + + - do: + bulk: + refresh: true + index: test-index1 + body: + - index: + _id: doc1 + - { "text": "document one" } + - index: + _id: doc2 + - { "text": "document two" } + - index: + _id: doc3 + - { "text": "document three" } + - index: + _id: doc4 + - { "text": "document four" } + - index: + _id: doc5 + - { "text": "document five" } + +--- +"rrf combined with pinned retriever": + - skip: { features: headers } + - do: + headers: + Content-Type: application/json + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1"] + retriever: + rrf: + retrievers: [ + { + standard: { + query: { + term: { text: "document" } + } + } + }, + { + standard: { + query: { + term: { text: "three" } + } + } + } + ] + rank_window_size: 10 + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014122E38 } + - match: { hits.hits.1._id: doc3 } + - match: { hits.hits.1._score < 100.0 } + - match: { hits.hits.2._id: doc2 } + +--- +"rrf with pinned retriever as a sub-retriever": + - skip: { features: headers } + - do: + headers: { Content-Type: application/json } + search: + index: test-index1 + body: + retriever: + rrf: + retrievers: + - + standard: + query: + match: { text: "document" } + - + pinned: + ids: ["doc4", "doc5"] + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - lt: { hits.hits.0._score: 100.0 } + - match: { hits.hits.1._id: doc4 } + - match: { hits.hits.2._id: doc5 } + + diff --git a/x-pack/plugin/search-business-rules/build.gradle b/x-pack/plugin/search-business-rules/build.gradle index cce6e15aedc06..98f79ee2f769b 100644 --- a/x-pack/plugin/search-business-rules/build.gradle +++ b/x-pack/plugin/search-business-rules/build.gradle @@ -1,5 +1,6 @@ apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' esplugin { name = 'search-business-rules' @@ -13,6 +14,19 @@ base { dependencies { compileOnly project(path: xpackModule('core')) + compileOnly project(':server') testImplementation(testArtifact(project(xpackModule('core')))) - testImplementation project(":test:framework") + testImplementation(testArtifact(project(':server'))) + clusterModules project(xpackModule('search-business-rules')) + clusterModules project(':modules:mapper-extras') + clusterModules project(':modules:lang-painless') + clusterModules project(xpackModule('inference')) } + +tasks.named("yamlRestTest") { + usesDefaultDistribution("uses search business rules plugin") +} +artifacts { + restXpackTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test")) +} + diff --git a/x-pack/plugin/search-business-rules/src/main/java/module-info.java b/x-pack/plugin/search-business-rules/src/main/java/module-info.java index d11c809b43854..aad2291cf82e5 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/module-info.java +++ b/x-pack/plugin/search-business-rules/src/main/java/module-info.java @@ -16,4 +16,7 @@ requires org.elasticsearch.xcore; exports org.elasticsearch.xpack.searchbusinessrules; + + provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRulesFeatures; + provides org.elasticsearch.plugins.SearchPlugin with org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRules; } diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java index c7a3b10cd7038..4b2d48ca1eaac 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java @@ -59,7 +59,7 @@ public class PinnedQueryBuilder extends AbstractQueryBuilder // Organic queries will have their scores capped to this number range, // We reserve the highest float exponent for scores of pinned queries - private static final float MAX_ORGANIC_SCORE = Float.intBitsToFloat((0xfe << 23)) - 1; + public static final float MAX_ORGANIC_SCORE = Float.intBitsToFloat((0xfe << 23)) - 1; public PinnedQueryBuilder(QueryBuilder organicQuery, String... ids) { this(organicQuery, Arrays.asList(ids), null); diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java index fcbb937e4b2a0..c96f93d55a940 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -7,18 +7,28 @@ package org.elasticsearch.xpack.searchbusinessrules; +import org.elasticsearch.plugins.ExtensiblePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.plugins.SearchPlugin.QuerySpec; +import org.elasticsearch.plugins.SearchPlugin.RetrieverSpec; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder; import java.util.List; import static java.util.Collections.singletonList; -public class SearchBusinessRules extends Plugin implements SearchPlugin { +public class SearchBusinessRules extends Plugin implements SearchPlugin, ExtensiblePlugin { @Override public List> getQueries() { - return singletonList(new QuerySpec<>(PinnedQueryBuilder.NAME, PinnedQueryBuilder::new, PinnedQueryBuilder::fromXContent)); + return List.of(new QuerySpec<>(PinnedQueryBuilder.NAME, PinnedQueryBuilder::new, PinnedQueryBuilder::fromXContent)); + } + + @Override + public List> getRetrievers() { + return singletonList(new RetrieverSpec<>(new ParseField(PinnedRetrieverBuilder.NAME), PinnedRetrieverBuilder::fromXContent)); } } diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java new file mode 100644 index 0000000000000..d702a0fcc6ea0 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.searchbusinessrules; + +import org.elasticsearch.features.FeatureSpecification; +import org.elasticsearch.features.NodeFeature; + +import java.util.Set; + +import static org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder.PINNED_RETRIEVER_FEATURE; + +public class SearchBusinessRulesFeatures implements FeatureSpecification { + + @Override + public Set getFeatures() { + return Set.of(); + } + + @Override + public Set getTestFeatures() { + return Set.of(PINNED_RETRIEVER_FEATURE); + } +} diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SpecifiedDocument.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SpecifiedDocument.java index 8c3801630fb2b..59d32ecf0b68a 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SpecifiedDocument.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SpecifiedDocument.java @@ -94,7 +94,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder.endObject(); } - static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( NAME, a -> new SpecifiedDocument((String) a[0], (String) a[1]) ); diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java new file mode 100644 index 0000000000000..fb13fad8720b1 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.searchbusinessrules.retriever; + +import org.apache.lucene.search.Explanation; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.search.rank.RankDoc; + +import java.io.IOException; +import java.util.Objects; + +public class PinnedRankDoc extends RankDoc { + public static final String NAME = "pinned_rank_doc"; + + private final boolean isPinned; + + public PinnedRankDoc(int docId, float score, int shardIndex, boolean isPinned) { + super(docId, score, shardIndex); + this.isPinned = isPinned; + } + + public PinnedRankDoc(StreamInput in) throws IOException { + super(in); + this.isPinned = in.readBoolean(); + } + + public boolean isPinned() { + return isPinned; + } + + @Override + public Explanation explain(Explanation[] sources, String[] queryNames) { + if (isPinned) { + return Explanation.match(score, "Pinned document, original explanation:", sources); + } else { + return super.explain(sources, queryNames); + } + } + + @Override + public String toString() { + return super.toString() + ", isPinned=" + isPinned; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeBoolean(isPinned); + } + + @Override + protected boolean doEquals(RankDoc rd) { + if (rd instanceof PinnedRankDoc other) { + return this.isPinned == other.isPinned; + } else { + return false; + } + } + + @Override + protected int doHashCode() { + return Objects.hash(super.doHashCode(), isPinned); + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.PINNED_RETRIEVER; + } +} diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java new file mode 100644 index 0000000000000..2e7c18e6a85b9 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -0,0 +1,270 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.searchbusinessrules.retriever; + +import org.apache.lucene.search.ScoreDoc; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RankDocsQueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.rank.RankDoc; +import org.elasticsearch.search.retriever.CompoundRetrieverBuilder; +import org.elasticsearch.search.retriever.RetrieverBuilder; +import org.elasticsearch.search.retriever.RetrieverBuilderWrapper; +import org.elasticsearch.search.retriever.RetrieverParserContext; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.ShardDocSortField; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContent.Params; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder; +import org.elasticsearch.xpack.searchbusinessrules.SpecifiedDocument; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE; +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * A pinned retriever applies pinned documents to the underlying retriever. + * This retriever will rewrite to a PinnedQueryBuilder. + */ +public final class PinnedRetrieverBuilder extends CompoundRetrieverBuilder { + + public static final String NAME = "pinned"; + + public static final ParseField IDS_FIELD = new ParseField("ids"); + public static final ParseField DOCS_FIELD = new ParseField("docs"); + public static final ParseField RETRIEVER_FIELD = new ParseField("retriever"); + + public static final NodeFeature PINNED_RETRIEVER_FEATURE = new NodeFeature("pinned_retriever_supported"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + NAME, + args -> { + List ids = (List) args[0]; + List docs = (List) args[1]; + RetrieverBuilder retrieverBuilder = (RetrieverBuilder) args[2]; + int rankWindowSize = args[3] == null ? DEFAULT_RANK_WINDOW_SIZE : (int) args[3]; + return new PinnedRetrieverBuilder(ids, docs, retrieverBuilder, rankWindowSize); + } + ); + + static { + PARSER.declareStringArray(optionalConstructorArg(), IDS_FIELD); + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> SpecifiedDocument.PARSER.apply(p, null), DOCS_FIELD); + PARSER.declareNamedObject(constructorArg(), (p, c, n) -> { + RetrieverBuilder innerRetriever = p.namedObject(RetrieverBuilder.class, n, c); + c.trackRetrieverUsage(innerRetriever.getName()); + return innerRetriever; + }, RETRIEVER_FIELD); + PARSER.declareInt(optionalConstructorArg(), RANK_WINDOW_SIZE_FIELD); + RetrieverBuilder.declareBaseParserFields(PARSER); + } + + public static PinnedRetrieverBuilder fromXContent(XContentParser parser, RetrieverParserContext context) throws IOException { + try { + return PARSER.apply(parser, context); + } catch (Exception e) { + throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e); + } + } + + private final List ids; + private final List docs; + + private void validateIdsAndDocs(List ids, List docs) { + if (ids != null && docs != null) { + throw new IllegalArgumentException("Both 'ids' and 'docs' cannot be specified at the same time"); + } + + boolean validIds = ids != null && ids.isEmpty() == false; + boolean validDocs = docs != null && docs.isEmpty() == false; + + if (validIds == false && validDocs == false) { + throw new IllegalArgumentException("Either 'ids' or 'docs' must be provided and non-empty for pinned retriever"); + } + } + + private void validateSort(SearchSourceBuilder source) { + List> sorts = source.sorts(); + if (sorts == null || sorts.isEmpty()) { + return; + } + for (SortBuilder sort : sorts) { + if (sort instanceof ScoreSortBuilder) { + continue; + } + if (sort instanceof FieldSortBuilder) { + FieldSortBuilder fieldSort = (FieldSortBuilder) sort; + if (ShardDocSortField.NAME.equals(fieldSort.getFieldName())) { + continue; + } + } + throw new IllegalArgumentException( + "[" + NAME + "] retriever only supports sorting by score, invalid sort criterion: " + sort.toString() + ); + } + } + + public PinnedRetrieverBuilder(List ids, List docs, RetrieverBuilder retrieverBuilder, int rankWindowSize) { + super(new ArrayList<>(), rankWindowSize); + validateIdsAndDocs(ids, docs); + this.ids = ids; + this.docs = docs; + addChild(new PinnedRetrieverBuilderWrapper(retrieverBuilder)); + } + + public PinnedRetrieverBuilder( + List ids, + List docs, + List retrieverSource, + int rankWindowSize, + String retrieverName, + List preFilterQueryBuilders + ) { + super(retrieverSource, rankWindowSize); + validateIdsAndDocs(ids, docs); + this.ids = ids; + this.docs = docs; + this.retrieverName = retrieverName; + this.preFilterQueryBuilders = preFilterQueryBuilders; + } + + @Override + public String getName() { + return NAME; + } + + public int rankWindowSize() { + return rankWindowSize; + } + + /** + * Creates a PinnedQueryBuilder with the appropriate pinned documents. + * + * @param baseQuery the base query to pin documents to + * @return a PinnedQueryBuilder + * @throws IllegalArgumentException if baseQuery is null + */ + private QueryBuilder createPinnedQuery(QueryBuilder baseQuery) { + Objects.requireNonNull(baseQuery, "pinned retriever requires retriever with associated query"); + + if (docs != null && docs.isEmpty() == false) { + return new PinnedQueryBuilder(baseQuery, docs.toArray(new SpecifiedDocument[0])); + } + return new PinnedQueryBuilder(baseQuery, ids.toArray(new String[0])); + } + + @Override + protected SearchSourceBuilder finalizeSourceBuilder(SearchSourceBuilder source) { + validateSort(source); + QueryBuilder underlyingQuery = source.query(); + if (underlyingQuery == null) { + throw new IllegalArgumentException("pinned retriever requires retriever with associated query"); + } + source.query(createPinnedQuery(underlyingQuery)); + return source; + } + + @Override + public void doToXContent(XContentBuilder builder, Params params) throws IOException { + if (ids != null) { + builder.array(IDS_FIELD.getPreferredName(), ids.toArray()); + } + if (docs != null) { + builder.startArray(DOCS_FIELD.getPreferredName()); + for (SpecifiedDocument doc : docs) { + builder.value(doc); + } + builder.endArray(); + } + builder.field(RETRIEVER_FIELD.getPreferredName(), innerRetrievers.getFirst().retriever()); + builder.field(RANK_WINDOW_SIZE_FIELD.getPreferredName(), rankWindowSize); + } + + @Override + protected PinnedRetrieverBuilder clone(List newChildRetrievers, List newPreFilterQueryBuilders) { + return new PinnedRetrieverBuilder(ids, docs, newChildRetrievers, rankWindowSize, retrieverName, newPreFilterQueryBuilders); + } + + @Override + protected RankDoc[] combineInnerRetrieverResults(List rankResults, boolean explain) { + assert rankResults.size() == 1; + ScoreDoc[] scoreDocs = rankResults.getFirst(); + RankDoc[] rankDocs = new RankDoc[scoreDocs.length]; + for (int i = 0; i < scoreDocs.length; i++) { + ScoreDoc scoreDoc = scoreDocs[i]; + + if (explain) { + boolean isPinned = scoreDoc.score > PinnedQueryBuilder.MAX_ORGANIC_SCORE; + if (isPinned) { + String pinnedBy = (this.ids != null && this.ids.isEmpty() == false) ? "ids" : "docs"; + rankDocs[i] = new PinnedRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex, true); + } else { + rankDocs[i] = new RankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex); + } + } else { + rankDocs[i] = new RankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex); + } + + rankDocs[i].rank = i + 1; + } + return rankDocs; + } + + @Override + public boolean doEquals(Object o) { + PinnedRetrieverBuilder that = (PinnedRetrieverBuilder) o; + return super.doEquals(o) && Objects.equals(ids, that.ids) && Objects.equals(docs, that.docs); + } + + @Override + public int doHashCode() { + return Objects.hash(super.doHashCode(), ids, docs); + } + + /** + * We need to wrap the PinnedRetrieverBuilder in order to ensure that the top docs query that is generated + * by this retriever correctly generates and executes a Pinned query. + */ + class PinnedRetrieverBuilderWrapper extends RetrieverBuilderWrapper { + protected PinnedRetrieverBuilderWrapper(RetrieverBuilder in) { + super(in); + } + + @Override + protected PinnedRetrieverBuilderWrapper clone(RetrieverBuilder in) { + return new PinnedRetrieverBuilderWrapper(in); + } + + @Override + public QueryBuilder topDocsQuery() { + return createPinnedQuery(in.topDocsQuery()); + } + + @Override + public QueryBuilder explainQuery() { + RankDoc[] currentRankDocs = in.getRankDocs(); + if (currentRankDocs == null) { + return in.explainQuery(); + } + return new RankDocsQueryBuilder(currentRankDocs, new QueryBuilder[] { in.explainQuery() }, true); + } + } +} diff --git a/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification new file mode 100644 index 0000000000000..04e9c67d68039 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -0,0 +1 @@ +org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRulesFeatures diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java new file mode 100644 index 0000000000000..38a2299ca26f9 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -0,0 +1,223 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.searchbusinessrules.retriever; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.retriever.RetrieverBuilder; +import org.elasticsearch.search.retriever.RetrieverParserContext; +import org.elasticsearch.search.retriever.TestRetrieverBuilder; +import org.elasticsearch.test.AbstractXContentTestCase; +import org.elasticsearch.usage.SearchUsage; +import org.elasticsearch.usage.SearchUsageHolder; +import org.elasticsearch.usage.UsageService; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.searchbusinessrules.SpecifiedDocument; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class PinnedRetrieverBuilderTests extends AbstractXContentTestCase { + + @Override + protected PinnedRetrieverBuilder createTestInstance() { + return createRandomPinnedRetrieverBuilder(); + } + + @Override + protected PinnedRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { + return (PinnedRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( + parser, + new RetrieverParserContext(new SearchUsage(), nf -> true) + ); + } + + public static PinnedRetrieverBuilder createRandomPinnedRetrieverBuilder() { + boolean useIds = randomBoolean(); + int numItems = randomIntBetween(1, 5); + + List ids = useIds + ? IntStream.range(0, numItems).mapToObj(i -> randomAlphaOfLengthBetween(5, 10)).collect(Collectors.toList()) + : null; + List docs = useIds + ? null + : IntStream.range(0, numItems) + .mapToObj(i -> new SpecifiedDocument(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10))) + .collect(Collectors.toList()); + return new PinnedRetrieverBuilder(ids, docs, TestRetrieverBuilder.createRandomTestRetrieverBuilder(), randomIntBetween(1, 100)); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + @Override + protected String[] getShuffleFieldsExceptions() { + return new String[] { PinnedRetrieverBuilder.IDS_FIELD.getPreferredName(), PinnedRetrieverBuilder.DOCS_FIELD.getPreferredName() }; + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + List entries = new SearchModule(Settings.EMPTY, List.of()).getNamedXContents(); + entries.add( + new NamedXContentRegistry.Entry( + RetrieverBuilder.class, + TestRetrieverBuilder.TEST_SPEC.getName(), + (p, c) -> TestRetrieverBuilder.TEST_SPEC.getParser().fromXContent(p, (RetrieverParserContext) c), + TestRetrieverBuilder.TEST_SPEC.getName().getForRestApiVersion() + ) + ); + entries.add( + new NamedXContentRegistry.Entry( + RetrieverBuilder.class, + new ParseField(PinnedRetrieverBuilder.NAME), + (p, c) -> PinnedRetrieverBuilder.PARSER.apply(p, (RetrieverParserContext) c) + ) + ); + return new NamedXContentRegistry(entries); + } + + public void testParserDefaults() throws IOException { + // Inner retriever content only sent to parser + String json = """ + { + "ids": [ "id1", "id2" ], + "retriever": { "standard": { "query": { "query_string": { "query": "i like pugs" } } } } + }"""; + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + PinnedRetrieverBuilder parsed = PinnedRetrieverBuilder.PARSER.parse( + parser, + new RetrieverParserContext(new SearchUsage(), nf -> true) + ); + assertEquals(DEFAULT_RANK_WINDOW_SIZE, parsed.rankWindowSize()); + } + } + + public void testPinnedRetrieverParsing() throws IOException { + String restContent = """ + { + "retriever": { + "pinned": { + "retriever": { + "test": { + "value": "my-test-retriever" + } + }, + "ids": [ + "id1", + "id2" + ], + "rank_window_size": 100, + "_name": "my_pinned_retriever" + } + } + }"""; + SearchUsageHolder searchUsageHolder = new UsageService().getSearchUsageHolder(); + try (XContentParser jsonParser = createParser(JsonXContent.jsonXContent, restContent)) { + SearchSourceBuilder source = new SearchSourceBuilder().parseXContent(jsonParser, true, searchUsageHolder, nf -> true); + assertThat(source.retriever(), instanceOf(PinnedRetrieverBuilder.class)); + PinnedRetrieverBuilder parsed = (PinnedRetrieverBuilder) source.retriever(); + assertThat(parsed.retrieverName(), equalTo("my_pinned_retriever")); + try (XContentParser parseSerialized = createParser(JsonXContent.jsonXContent, Strings.toString(source))) { + SearchSourceBuilder deserializedSource = new SearchSourceBuilder().parseXContent( + parseSerialized, + true, + searchUsageHolder, + nf -> true + ); + assertThat(deserializedSource.retriever(), instanceOf(PinnedRetrieverBuilder.class)); + PinnedRetrieverBuilder deserialized = (PinnedRetrieverBuilder) deserializedSource.retriever(); + assertThat(parsed, equalTo(deserialized)); + } + } + } + + public void testValidation() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + new PinnedRetrieverBuilder( + List.of("id1"), + List.of(new SpecifiedDocument("id2", "index")), + new TestRetrieverBuilder("test"), + DEFAULT_RANK_WINDOW_SIZE + ); + }); + assertThat(e.getMessage(), equalTo("Both 'ids' and 'docs' cannot be specified at the same time")); + + e = expectThrows(IllegalArgumentException.class, () -> { + new PinnedRetrieverBuilder(List.of(), List.of(), new TestRetrieverBuilder("test"), DEFAULT_RANK_WINDOW_SIZE); + }); + assertThat(e.getMessage(), equalTo("Both 'ids' and 'docs' cannot be specified at the same time")); + } + + public void testValidateSort() { + + PinnedRetrieverBuilder builder = new PinnedRetrieverBuilder( + List.of("id1"), + null, + new TestRetrieverBuilder("test"), + DEFAULT_RANK_WINDOW_SIZE + ); + + QueryBuilder dummyQuery = new MatchAllQueryBuilder(); + + SearchSourceBuilder emptySource = new SearchSourceBuilder(); + emptySource.query(dummyQuery); + builder.finalizeSourceBuilder(emptySource); + assertThat(emptySource.sorts(), equalTo(null)); + + SearchSourceBuilder scoreSource = new SearchSourceBuilder(); + scoreSource.query(dummyQuery); + scoreSource.sort("_score"); + builder.finalizeSourceBuilder(scoreSource); + assertThat(scoreSource.sorts().size(), equalTo(1)); + + SearchSourceBuilder customSortSource = new SearchSourceBuilder(); + customSortSource.query(dummyQuery); + customSortSource.sort("field1"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.finalizeSourceBuilder(customSortSource)); + assertThat( + e.getMessage(), + equalTo( + "[" + + PinnedRetrieverBuilder.NAME + + "] retriever only supports sorting by score, " + + "invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}" + ) + ); + + SearchSourceBuilder multipleSortsSource = new SearchSourceBuilder(); + multipleSortsSource.query(dummyQuery); + multipleSortsSource.sort("_score"); + multipleSortsSource.sort("field1"); + e = expectThrows(IllegalArgumentException.class, () -> builder.finalizeSourceBuilder(multipleSortsSource)); + assertThat( + e.getMessage(), + equalTo( + "[" + + PinnedRetrieverBuilder.NAME + + "] retriever only supports sorting by score, " + + "invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}" + ) + ); + } +} diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..59971598269fa --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.searchbusinessrules; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; + +public class SearchBusinessRulesClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .nodes(1) + .module("search-business-rules") + .module("mapper-extras") + .module("lang-painless") + .module("x-pack-inference") + .setting("xpack.license.self_generated.type", "basic") + .setting("xpack.security.enabled", "false") + .build(); + + public SearchBusinessRulesClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(new String[] { "search-business-rules/10_pinned_retriever" }); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml new file mode 100644 index 0000000000000..d052e5ae847f3 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -0,0 +1,511 @@ +setup: + - requires: + cluster_features: 'pinned_retriever_supported' + reason: 'test requires pinned retriever implementation' + - do: + indices.create: + index: test-index1 + - do: + indices.create: + index: test-index2 + + - do: + bulk: + refresh: true + index: test-index1 + body: + - index: + _id: doc1 + - { "text": "document one" } + - index: + _id: doc2 + - { "text": "document two" } + - index: + _id: doc3 + - { "text": "document three" } + - index: + _id: doc4 + - { "text": "document four" } + - index: + _id: doc5 + - { "text": "document five" } + - do: + bulk: + refresh: true + index: test-index2 + body: + - index: + _id: idx2-docA + - { "text": "index two document A" } + - index: + _id: idx2-docB + - { "text": "index two document B" } + +--- +"pinned retriever parameter variations": + - skip: { features: headers } + - do: + headers: { Content-Type: application/json } + search: + index: test-index1 + body: + retriever: + pinned: + docs: + - _index: test-index1 + _id: doc1 + - _index: test-index1 + _id: doc2 + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014124E38 } + - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.1._score: 1.7014122E38 } + - match: { hits.hits.2._id: doc3 } + - match: { hits.hits.2._score < 100.0 } + + - do: + headers: { Content-Type: application/json } + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1"] + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014122E38 } + - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.1._score < 100.0 } + + - do: + headers: { Content-Type: application/json } + search: + index: test-index1 + body: + retriever: + pinned: + docs: + - _index: test-index1 + _id: doc1 + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014122E38 } + - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.1._score < 100.0 } + +--- +"pinned retriever dynamic pinning and ordering": + - skip: + features: headers + - do: + headers: + Content-Type: application/json + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014124E38 } + - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.1._score: 1.7014122E38 } + - match: { hits.hits.2._id: doc3 } + - match: { hits.hits.2._score < 100.0 } + + - do: + headers: + Content-Type: application/json + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc3"] + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014124E38 } + - match: { hits.hits.1._id: doc3 } + - match: { hits.hits.1._score: 1.7014122E38 } + - match: { hits.hits.2._id: doc2 } + - match: { hits.hits.2._score < 100.0 } + - match: { hits.hits.3._id: doc4 } + - match: { hits.hits.4._id: doc5 } + +--- +"pinned retriever with pagination": + - skip: { features: headers } + - do: + headers: { Content-Type: application/json } + search: + index: test-index1 + body: + size: 1 + from: 1 + retriever: + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 5 } + - length: { hits.hits: 1 } + - match: { hits.hits.0._id: doc2 } + - match: { hits.hits.0._score: 1.7014122E38 } + +--- +"pinned retriever with explicit sort on score": + - skip: { features: headers } + - do: + headers: { Content-Type: application/json } + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: + match: { text: "document" } + sort: [ "_score" ] + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014124E38 } + - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.1._score: 1.7014122E38 } + - match: { hits.hits.2._id: doc3 } + - match: { hits.hits.2._score < 100.0 } + +--- +"pinned retriever with rank window size": + - skip: + features: headers + + - do: + headers: + Content-Type: application/json + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: + match: { text: "document" } + rank_window_size: 1 + + - match: { hits.total.value: 5 } + - length: { hits.hits: 1 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014124E38 } + + - do: + headers: + Content-Type: application/json + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: + match: { text: "document" } + rank_window_size: 10 + + - match: { hits.total.value: 5 } + - length: { hits.hits: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014124E38 } + - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.1._score: 1.7014122E38 } + - match: { hits.hits.2._id: doc3 } + - match: { hits.hits.2._score < 100.0 } + +--- +"pinned retriever explanation": + - skip: { features: headers } + + - do: + headers: + # Force JSON content type so that we use a parser that interprets the floating-point score as a double + Content-Type: application/json + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: + match: { text: "document" } + explain: true + + - match: { hits.hits.0._id: doc1 } + - gt: { hits.hits.0._score: 1.0 } + - match: + hits.hits.0._explanation.description: "Pinned document, original explanation:" + - match: + hits.hits.0._explanation.details.0.description: "doc [0] with an original score of [1.7014124E38] is at rank [1] from the following source queries." + + - match: { hits.hits.1._id: doc2 } + - gt: { hits.hits.1._score: 1.0 } + - match: + hits.hits.1._explanation.description: "Pinned document, original explanation:" + - match: + hits.hits.1._explanation.details.0.description: "doc [1] with an original score of [1.7014122E38] is at rank [2] from the following source queries." + +--- +"pinned retriever with empty parameters": + - do: + catch: /Either 'ids' or 'docs' must be provided and non-empty for pinned retriever/ + search: + index: test-index1 + body: + retriever: + pinned: + retriever: + standard: + query: + match: { text: "document" } + +--- +"pinned retriever error case - both ids and docs": + - do: + catch: /Both 'ids' and 'docs' cannot be specified at the same time/ + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1"] + docs: + - _index: test-index1 + _id: doc2 + retriever: + standard: + query: + match: { text: "document" } + +--- +"pinned retriever error case - duplicate id": + - do: + catch: /duplicate id found/ + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc1"] + retriever: + standard: + query: + match: { text: "document" } + +--- +"pinned retriever error case - duplicate doc": + - do: + catch: /duplicate doc found/ + search: + index: test-index1 + body: + retriever: + pinned: + docs: + - _index: test-index1 + _id: doc1 + - _index: test-index1 + _id: doc1 + retriever: + standard: + query: + match: { text: "document" } + +--- +"pinned retriever with ids and empty docs array should fail": + - do: + catch: /Both 'ids' and 'docs' cannot be specified at the same time/ + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1"] + docs: [] + retriever: + standard: + query: + match: { text: "document" } + +--- +"pinned retriever errors if sorting by anything other than score": + - do: + catch: /\[pinned\] retriever only supports sorting by score/ + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1"] + retriever: + standard: + query: + match: { text: "document" } + sort: [ { "_id": "desc" } ] + +--- +"pinned retriever error case - null inner query": + - do: + catch: /illegal_argument_exception.+reason=pinned retriever requires retriever with associated query/ + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1"] + retriever: + standard: {} + +--- +"pinned retriever with non-existent id": + - skip: { features: headers } + - do: + headers: { Content-Type: application/json } + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "nonexistent_doc", "doc3"] + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014126E38 } + - match: { hits.hits.1._id: doc3 } + - match: { hits.hits.1._score: 1.7014122E38 } + - match: { hits.hits.2._id: doc2 } + - match: { hits.hits.2._score < 100.0 } + +--- +"pinned retriever with non-existent doc specified via docs": + - skip: { features: headers } + - do: + headers: { Content-Type: application/json } + search: + index: test-index1 + body: + retriever: + pinned: + docs: + - _index: test-index1 + _id: doc1 + - _index: test-index1 + _id: nonexistent_doc + - _index: test-index1 + _id: doc3 + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014126E38 } + - match: { hits.hits.1._id: doc3 } + - match: { hits.hits.1._score: 1.7014122E38 } + - match: { hits.hits.2._id: doc2 } + - match: { hits.hits.2._score < 100.0 } + - match: { hits.hits.3._id: doc4 } + - match: { hits.hits.4._id: doc5 } + +--- +"pinned retriever multi-index vs single-index filtering": + - skip: { features: headers } + + - do: + headers: { Content-Type: application/json } + search: + index: test-index1,test-index2 + body: + retriever: + pinned: + ids: ["doc1", "idx2-docA"] + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 7 } + - length: { hits.hits: 7 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.1._id: idx2-docA } + - match: { hits.hits.2._id: idx2-docB } + - match: { hits.hits.3._id: doc2 } + - match: { hits.hits.4._id: doc3 } + - match: { hits.hits.5._id: doc4 } + - match: { hits.hits.6._id: doc5 } + + - do: + headers: { Content-Type: application/json } + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "idx2-docA"] + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 5 } + - length: { hits.hits: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014124E38 } + - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.2._id: doc3 } + - match: { hits.hits.3._id: doc4 } + - match: { hits.hits.4._id: doc5 } + + + +