From 84b6ed7ae8fa00e051c41b654d65f0613bcc1db6 Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 7 Apr 2025 09:23:23 +0100 Subject: [PATCH 01/83] Created PinnedQueryRetriever Builder --- .../retriever/PinnedRetrieverBuilder.java | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java new file mode 100644 index 0000000000000..d201dfd4188f4 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -0,0 +1,221 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.searchbusinessrules.retriever; + +import org.apache.lucene.search.ScoreDoc; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.rank.RankDoc; +import org.elasticsearch.search.retriever.CompoundRetrieverBuilder; +import org.elasticsearch.search.retriever.RetrieverBuilder; +import org.elasticsearch.search.retriever.RetrieverBuilderWrapper; +import org.elasticsearch.search.retriever.RetrieverParserContext; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder; +import org.elasticsearch.xpack.searchbusinessrules.SpecifiedDocument; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE; +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * A pinned retriever applies pinned documents to the underlying retriever. + * This retriever will rewrite to a PinnedQueryBuilder. + */ +public final class PinnedRetrieverBuilder extends CompoundRetrieverBuilder { + + public static final String NAME = "pinned"; + + public static final ParseField IDS_FIELD = new ParseField("ids"); + public static final ParseField DOCS_FIELD = new ParseField("docs"); + public static final ParseField RETRIEVER_FIELD = new ParseField("retriever"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + NAME, + args -> { + List ids = (List) args[0]; + List docs = (List) args[1]; + RetrieverBuilder retrieverBuilder = (RetrieverBuilder) args[2]; + int rankWindowSize = args[3] == null ? DEFAULT_RANK_WINDOW_SIZE : (int) args[3]; + return new PinnedRetrieverBuilder(ids, docs, retrieverBuilder, rankWindowSize); + } + ); + + static { + PARSER.declareStringArray(optionalConstructorArg(), IDS_FIELD); + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> { + String index = p.textOrNull(); + String id = p.text(); + return new SpecifiedDocument(index, id); + }, DOCS_FIELD); + PARSER.declareNamedObject(constructorArg(), (p, c, n) -> { + RetrieverBuilder innerRetriever = p.namedObject(RetrieverBuilder.class, n, c); + c.trackRetrieverUsage(innerRetriever.getName()); + return innerRetriever; + }, RETRIEVER_FIELD); + PARSER.declareInt(optionalConstructorArg(), RANK_WINDOW_SIZE_FIELD); + RetrieverBuilder.declareBaseParserFields(PARSER); + } + + public static PinnedRetrieverBuilder fromXContent(XContentParser parser, RetrieverParserContext context) throws IOException { + try { + return PARSER.apply(parser, context); + } catch (Exception e) { + throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e); + } + } + + private final List ids; + private final List docs; + + public PinnedRetrieverBuilder( + List ids, + List docs, + RetrieverBuilder retrieverBuilder, + int rankWindowSize + ) { + super(new ArrayList<>(), rankWindowSize); + this.ids = ids != null ? ids : new ArrayList<>(); + this.docs = docs != null ? docs : new ArrayList<>(); + addChild(new PinnedRetrieverBuilderWrapper(retrieverBuilder)); + } + + public PinnedRetrieverBuilder( + List ids, + List docs, + List retrieverSource, + int rankWindowSize, + String retrieverName, + List preFilterQueryBuilders + ) { + super(retrieverSource, rankWindowSize); + this.ids = ids; + this.docs = docs; + this.retrieverName = retrieverName; + this.preFilterQueryBuilders = preFilterQueryBuilders; + } + + @Override + public String getName() { + return NAME; + } + + public int rankWindowSize() { + return rankWindowSize; + } + + /** + * Creates a PinnedQueryBuilder with the appropriate pinned documents. + * Prioritizes docs over ids if both are present. + * + * @param baseQuery the base query to pin documents to + * @return a PinnedQueryBuilder or the original query if no pinned documents + */ + private QueryBuilder createPinnedQuery(QueryBuilder baseQuery) { + if (!docs.isEmpty()) { + return new PinnedQueryBuilder(baseQuery, docs.toArray(new SpecifiedDocument[0])); + } else if (!ids.isEmpty()) { + return new PinnedQueryBuilder(baseQuery, ids.toArray(new String[0])); + } else { + return baseQuery; + } + } + + @Override + protected SearchSourceBuilder finalizeSourceBuilder(SearchSourceBuilder source) { + source.query(createPinnedQuery(source.query())); + return source; + } + + @Override + public void doToXContent(XContentBuilder builder, Params params) throws IOException { + if (ids != null && !ids.isEmpty()) { + builder.array(IDS_FIELD.getPreferredName(), ids.toArray()); + } + if (docs != null && !docs.isEmpty()) { + builder.startArray(DOCS_FIELD.getPreferredName()); + for (SpecifiedDocument doc : docs) { + builder.value(doc); + } + builder.endArray(); + } + builder.field(RETRIEVER_FIELD.getPreferredName(), innerRetrievers.getFirst().retriever()); + builder.field(RANK_WINDOW_SIZE_FIELD.getPreferredName(), rankWindowSize); + } + + @Override + protected PinnedRetrieverBuilder clone(List newChildRetrievers, List newPreFilterQueryBuilders) { + return new PinnedRetrieverBuilder( + ids, + docs, + newChildRetrievers, + rankWindowSize, + retrieverName, + newPreFilterQueryBuilders + ); + } + + @Override + protected RankDoc[] combineInnerRetrieverResults(List rankResults, boolean explain) { + assert rankResults.size() == 1; + ScoreDoc[] scoreDocs = rankResults.getFirst(); + RankDoc[] rankDocs = new RankDoc[scoreDocs.length]; + for (int i = 0; i < scoreDocs.length; i++) { + ScoreDoc scoreDoc = scoreDocs[i]; + rankDocs[i] = new RankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex); + rankDocs[i].rank = i + 1; + } + return rankDocs; + } + + @Override + public boolean doEquals(Object o) { + PinnedRetrieverBuilder that = (PinnedRetrieverBuilder) o; + return super.doEquals(o) && Objects.equals(ids, that.ids) && Objects.equals(docs, that.docs); + } + + @Override + public int doHashCode() { + return Objects.hash(super.doHashCode(), ids, docs); + } + + /** + * We need to wrap the PinnedRetrieverBuilder in order to ensure that the top docs query that is generated + * by this retriever correctly generates and executes a Pinned query. + */ + class PinnedRetrieverBuilderWrapper extends RetrieverBuilderWrapper { + protected PinnedRetrieverBuilderWrapper(RetrieverBuilder in) { + super(in); + } + + @Override + protected PinnedRetrieverBuilderWrapper clone(RetrieverBuilder in) { + return new PinnedRetrieverBuilderWrapper(in); + } + + @Override + public QueryBuilder topDocsQuery() { + return createPinnedQuery(in.topDocsQuery()); + } + + @Override + public QueryBuilder explainQuery() { + return createPinnedQuery(in.explainQuery()); + } + } +} \ No newline at end of file From 4c6f2849f36607515b8e101e2cecf7adba554973 Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 7 Apr 2025 11:29:54 +0100 Subject: [PATCH 02/83] Added retriever build test file --- .../SpecifiedDocument.java | 2 +- .../retriever/PinnedRetrieverBuilder.java | 6 +- .../PinnedRetrieverBuilderTests.java | 173 ++++++++++++++++++ 3 files changed, 175 insertions(+), 6 deletions(-) create mode 100644 x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SpecifiedDocument.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SpecifiedDocument.java index 8c3801630fb2b..59d32ecf0b68a 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SpecifiedDocument.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SpecifiedDocument.java @@ -94,7 +94,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder.endObject(); } - static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( NAME, a -> new SpecifiedDocument((String) a[0], (String) a[1]) ); diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index d201dfd4188f4..d86efbc3a8475 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -58,11 +58,7 @@ public final class PinnedRetrieverBuilder extends CompoundRetrieverBuilder { - String index = p.textOrNull(); - String id = p.text(); - return new SpecifiedDocument(index, id); - }, DOCS_FIELD); + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> SpecifiedDocument.PARSER.apply(p, null), DOCS_FIELD); PARSER.declareNamedObject(constructorArg(), (p, c, n) -> { RetrieverBuilder innerRetriever = p.namedObject(RetrieverBuilder.class, n, c); c.trackRetrieverUsage(innerRetriever.getName()); diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java new file mode 100644 index 0000000000000..5499c6d45572c --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -0,0 +1,173 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.searchbusinessrules.retriever; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.retriever.RetrieverBuilder; +import org.elasticsearch.search.retriever.RetrieverParserContext; +import org.elasticsearch.search.retriever.TestRetrieverBuilder; +import org.elasticsearch.test.AbstractXContentTestCase; +import org.elasticsearch.usage.SearchUsage; +import org.elasticsearch.usage.SearchUsageHolder; +import org.elasticsearch.usage.UsageService; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder; +import org.elasticsearch.xpack.searchbusinessrules.SpecifiedDocument; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class PinnedRetrieverBuilderTests extends AbstractXContentTestCase { + + public static PinnedRetrieverBuilder createRandomPinnedRetrieverBuilder() { + boolean useIds = randomBoolean(); + boolean useDocs = !useIds || randomBoolean(); + + List ids = useIds ? List.of(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)) : new ArrayList<>(); + List docs = useDocs ? List.of( + new SpecifiedDocument(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)), + new SpecifiedDocument(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)) + ) : new ArrayList<>(); + + return new PinnedRetrieverBuilder( + ids, + docs, + TestRetrieverBuilder.createRandomTestRetrieverBuilder(), + randomIntBetween(1, 100) + ); + } + + @Override + protected PinnedRetrieverBuilder createTestInstance() { + return createRandomPinnedRetrieverBuilder(); + } + + @Override + protected PinnedRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { + return (PinnedRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( + parser, + new RetrieverParserContext( + new SearchUsage(), + nf -> false + ) + ); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + @Override + protected String[] getShuffleFieldsExceptions() { + return new String[] { + PinnedRetrieverBuilder.IDS_FIELD.getPreferredName(), + PinnedRetrieverBuilder.DOCS_FIELD.getPreferredName() }; + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + List entries = new SearchModule(Settings.EMPTY, List.of()).getNamedXContents(); + entries.add( + new NamedXContentRegistry.Entry( + RetrieverBuilder.class, + TestRetrieverBuilder.TEST_SPEC.getName(), + (p, c) -> TestRetrieverBuilder.TEST_SPEC.getParser().fromXContent(p, (RetrieverParserContext) c), + TestRetrieverBuilder.TEST_SPEC.getName().getForRestApiVersion() + ) + ); + entries.add( + new NamedXContentRegistry.Entry( + RetrieverBuilder.class, + new ParseField(PinnedRetrieverBuilder.NAME), + (p, c) -> PinnedRetrieverBuilder.PARSER.apply(p, (RetrieverParserContext) c) + ) + ); + return new NamedXContentRegistry(entries); + } + + public void testParserDefaults() throws IOException { + // Inner retriever content only sent to parser + String json = """ + { + "ids": [ "id1", "id2" ], + "retriever": { "standard": { "query": { "query_string": { "query": "i like pugs" } } } } + }"""; + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + PinnedRetrieverBuilder parsed = PinnedRetrieverBuilder.PARSER.parse( + parser, + new RetrieverParserContext(new SearchUsage(), nf -> true) + ); + assertEquals(DEFAULT_RANK_WINDOW_SIZE, parsed.rankWindowSize()); + } + } + + public void testPinnedRetrieverParsing() throws IOException { + String restContent = """ + { + "retriever": { + "pinned": { + "retriever": { + "test": { + "value": "my-test-retriever" + } + }, + "ids": [ + "id1", + "id2" + ], + "docs": [ + { + "_index": "index1", + "_id": "doc1" + }, + { + "_index": "index2", + "_id": "doc2" + } + ], + "rank_window_size": 100, + "_name": "my_pinned_retriever" + } + } + }"""; + SearchUsageHolder searchUsageHolder = new UsageService().getSearchUsageHolder(); + try (XContentParser jsonParser = createParser(JsonXContent.jsonXContent, restContent)) { + SearchSourceBuilder source = new SearchSourceBuilder().parseXContent(jsonParser, true, searchUsageHolder, nf -> true); + assertThat(source.retriever(), instanceOf(PinnedRetrieverBuilder.class)); + PinnedRetrieverBuilder parsed = (PinnedRetrieverBuilder) source.retriever(); + assertThat(parsed.retrieverName(), equalTo("my_pinned_retriever")); + try (XContentParser parseSerialized = createParser(JsonXContent.jsonXContent, Strings.toString(source))) { + SearchSourceBuilder deserializedSource = new SearchSourceBuilder().parseXContent( + parseSerialized, + true, + searchUsageHolder, + nf -> true + ); + assertThat(deserializedSource.retriever(), instanceOf(PinnedRetrieverBuilder.class)); + PinnedRetrieverBuilder deserialized = (PinnedRetrieverBuilder) deserializedSource.retriever(); + assertThat(parsed, equalTo(deserialized)); + } + } + } +} \ No newline at end of file From 18bf6e4eaf8ee9c0a884ea4e77eb56d2e17d9217 Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 7 Apr 2025 12:02:30 +0100 Subject: [PATCH 03/83] Changed the test to accept all the retrievers as its not licensed --- .../retriever/PinnedRetrieverBuilderTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java index 5499c6d45572c..5de7d91b934a1 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -67,7 +67,7 @@ protected PinnedRetrieverBuilder doParseInstance(XContentParser parser) throws I parser, new RetrieverParserContext( new SearchUsage(), - nf -> false + nf -> true ) ); } From 8bab3d2fc5f4626c2783603ca77862452ca0d9c1 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 7 Apr 2025 12:32:00 +0000 Subject: [PATCH 04/83] [CI] Auto commit changes from spotless --- .../retriever/PinnedRetrieverBuilder.java | 18 ++-------- .../PinnedRetrieverBuilderTests.java | 36 +++++++------------ 2 files changed, 15 insertions(+), 39 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index d86efbc3a8475..77161d81116ce 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -79,12 +79,7 @@ public static PinnedRetrieverBuilder fromXContent(XContentParser parser, Retriev private final List ids; private final List docs; - public PinnedRetrieverBuilder( - List ids, - List docs, - RetrieverBuilder retrieverBuilder, - int rankWindowSize - ) { + public PinnedRetrieverBuilder(List ids, List docs, RetrieverBuilder retrieverBuilder, int rankWindowSize) { super(new ArrayList<>(), rankWindowSize); this.ids = ids != null ? ids : new ArrayList<>(); this.docs = docs != null ? docs : new ArrayList<>(); @@ -156,14 +151,7 @@ public void doToXContent(XContentBuilder builder, Params params) throws IOExcept @Override protected PinnedRetrieverBuilder clone(List newChildRetrievers, List newPreFilterQueryBuilders) { - return new PinnedRetrieverBuilder( - ids, - docs, - newChildRetrievers, - rankWindowSize, - retrieverName, - newPreFilterQueryBuilders - ); + return new PinnedRetrieverBuilder(ids, docs, newChildRetrievers, rankWindowSize, retrieverName, newPreFilterQueryBuilders); } @Override @@ -214,4 +202,4 @@ public QueryBuilder explainQuery() { return createPinnedQuery(in.explainQuery()); } } -} \ No newline at end of file +} diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java index 5de7d91b934a1..0d139d8335efe 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -9,8 +9,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.retriever.RetrieverBuilder; @@ -24,11 +22,9 @@ import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.json.JsonXContent; -import org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder; import org.elasticsearch.xpack.searchbusinessrules.SpecifiedDocument; import java.io.IOException; -import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; @@ -41,19 +37,16 @@ public class PinnedRetrieverBuilderTests extends AbstractXContentTestCase ids = useIds ? List.of(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)) : new ArrayList<>(); - List docs = useDocs ? List.of( - new SpecifiedDocument(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)), - new SpecifiedDocument(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)) - ) : new ArrayList<>(); - - return new PinnedRetrieverBuilder( - ids, - docs, - TestRetrieverBuilder.createRandomTestRetrieverBuilder(), - randomIntBetween(1, 100) - ); + List docs = useDocs + ? List.of( + new SpecifiedDocument(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)), + new SpecifiedDocument(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)) + ) + : new ArrayList<>(); + + return new PinnedRetrieverBuilder(ids, docs, TestRetrieverBuilder.createRandomTestRetrieverBuilder(), randomIntBetween(1, 100)); } @Override @@ -65,10 +58,7 @@ protected PinnedRetrieverBuilder createTestInstance() { protected PinnedRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { return (PinnedRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( parser, - new RetrieverParserContext( - new SearchUsage(), - nf -> true - ) + new RetrieverParserContext(new SearchUsage(), nf -> true) ); } @@ -79,9 +69,7 @@ protected boolean supportsUnknownFields() { @Override protected String[] getShuffleFieldsExceptions() { - return new String[] { - PinnedRetrieverBuilder.IDS_FIELD.getPreferredName(), - PinnedRetrieverBuilder.DOCS_FIELD.getPreferredName() }; + return new String[] { PinnedRetrieverBuilder.IDS_FIELD.getPreferredName(), PinnedRetrieverBuilder.DOCS_FIELD.getPreferredName() }; } @Override @@ -170,4 +158,4 @@ public void testPinnedRetrieverParsing() throws IOException { } } } -} \ No newline at end of file +} From 2cfcdc1eb650401a1f7d23bffa3bac9a24c20a92 Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 7 Apr 2025 17:24:11 +0100 Subject: [PATCH 05/83] Added integration test and fixed the code style issues --- .../retriever/PinnedRetrieverBuilder.java | 8 +- .../PinnedRetrieverBuilderTests.java | 14 +- .../10_pinned_retriever.yml | 386 ++++++++++++++++++ 3 files changed, 403 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index 77161d81116ce..753d9c4e97b92 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -118,9 +118,9 @@ public int rankWindowSize() { * @return a PinnedQueryBuilder or the original query if no pinned documents */ private QueryBuilder createPinnedQuery(QueryBuilder baseQuery) { - if (!docs.isEmpty()) { + if (docs.isEmpty() == false) { return new PinnedQueryBuilder(baseQuery, docs.toArray(new SpecifiedDocument[0])); - } else if (!ids.isEmpty()) { + } else if (ids.isEmpty() == false) { return new PinnedQueryBuilder(baseQuery, ids.toArray(new String[0])); } else { return baseQuery; @@ -135,10 +135,10 @@ protected SearchSourceBuilder finalizeSourceBuilder(SearchSourceBuilder source) @Override public void doToXContent(XContentBuilder builder, Params params) throws IOException { - if (ids != null && !ids.isEmpty()) { + if (ids != null && ids.isEmpty() == false) { builder.array(IDS_FIELD.getPreferredName(), ids.toArray()); } - if (docs != null && !docs.isEmpty()) { + if (docs != null && docs.isEmpty() == false) { builder.startArray(DOCS_FIELD.getPreferredName()); for (SpecifiedDocument doc : docs) { builder.value(doc); diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java index 0d139d8335efe..53d1edfcabba8 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -36,7 +36,7 @@ public class PinnedRetrieverBuilderTests extends AbstractXContentTestCase ids = useIds ? List.of(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)) : new ArrayList<>(); List docs = useDocs @@ -47,6 +47,14 @@ public static PinnedRetrieverBuilder createRandomPinnedRetrieverBuilder() { : new ArrayList<>(); return new PinnedRetrieverBuilder(ids, docs, TestRetrieverBuilder.createRandomTestRetrieverBuilder(), randomIntBetween(1, 100)); + List docs = useDocs + ? List.of( + new SpecifiedDocument(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)), + new SpecifiedDocument(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)) + ) + : new ArrayList<>(); + + return new PinnedRetrieverBuilder(ids, docs, TestRetrieverBuilder.createRandomTestRetrieverBuilder(), randomIntBetween(1, 100)); } @Override @@ -59,6 +67,7 @@ protected PinnedRetrieverBuilder doParseInstance(XContentParser parser) throws I return (PinnedRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( parser, new RetrieverParserContext(new SearchUsage(), nf -> true) + new RetrieverParserContext(new SearchUsage(), nf -> true) ); } @@ -70,6 +79,7 @@ protected boolean supportsUnknownFields() { @Override protected String[] getShuffleFieldsExceptions() { return new String[] { PinnedRetrieverBuilder.IDS_FIELD.getPreferredName(), PinnedRetrieverBuilder.DOCS_FIELD.getPreferredName() }; + return new String[] { PinnedRetrieverBuilder.IDS_FIELD.getPreferredName(), PinnedRetrieverBuilder.DOCS_FIELD.getPreferredName() }; } @Override @@ -159,3 +169,5 @@ public void testPinnedRetrieverParsing() throws IOException { } } } + +} diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml new file mode 100644 index 0000000000000..c10649b1e10e5 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml @@ -0,0 +1,386 @@ +setup: + - do: + indices.create: + index: test-index1 + body: + settings: + index: + number_of_shards: 1 + number_of_replicas: 0 + + - do: + bulk: + refresh: true + index: test-index1 + body: + - index: + _id: doc1 + - { "text": "document one" } + - index: + _id: doc2 + - { "text": "document two" } + - index: + _id: doc3 + - { "text": "document three" } + - index: + _id: doc4 + - { "text": "document four" } + - index: + _id: doc5 + - { "text": "document five" } + +--- +"pinned retriever basic functionality": + + - do: + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: + match_all: {} + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.2._id: doc3 } + - match: { hits.hits.3._id: doc4 } + - match: { hits.hits.4._id: doc5 } + +--- +"pinned retriever parameter variations": + + - do: + search: + index: test-index1 + body: + retriever: + pinned: + docs: + - index: test-index1 + id: doc1 + - index: test-index1 + id: doc2 + retriever: + standard: + query: + match_all: {} + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.1._id: doc2 } + + - do: + search: + index: test-index1 + body: + retriever: + pinned: + id: "doc1" + retriever: + standard: + query: + match_all: {} + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + + - do: + search: + index: test-index1 + body: + retriever: + pinned: + doc: + index: test-index1 + id: doc1 + retriever: + standard: + query: + match_all: {} + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + +--- +"pinned retriever with combined parameters": + + - do: + search: + index: test-index1 + body: + retriever: + pinned: + id: "doc1" + doc: + index: test-index1 + id: doc2 + retriever: + standard: + query: + match_all: {} + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.1._id: doc2 } + + - do: + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc3"] + docs: + - index: test-index1 + id: doc2 + - index: test-index1 + id: doc4 + retriever: + standard: + query: + match_all: {} + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.1._id: doc3 } + - match: { hits.hits.2._id: doc2 } + - match: { hits.hits.3._id: doc4 } + +--- +"pinned retriever combined with rrf": + + - do: + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc2"] + retriever: + rrf: + retrievers: [ + { + standard: { + query: { + match_all: {} + } + } + }, + { + standard: { + query: { + match_all: {} + } + } + } + ] + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.1._id: doc2 } + +--- +"pinned retriever with pagination": + + - do: + search: + index: test-index1 + body: + size: 1 + from: 1 + retriever: + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: + match_all: {} + + - match: { hits.total.value: 5 } + - length: { hits.hits: 1 } + - match: { hits.hits.0._id: doc2 } + +--- +"pinned retriever as a sub-retriever": + + - do: + search: + index: test-index1 + body: + retriever: + rrf: + retrievers: [ + { + standard: { + query: { + match_all: {} + } + } + }, + { + pinned: { + ids: ["doc1", "doc2"], + retriever: { + standard: { + query: { + match_all: {} + } + } + } + } + } + ] + + - match: { hits.total.value: 5 } + +--- +"pinned retriever with explicit sort on score": + + - do: + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: + match_all: {} + sort: [ "_score" ] + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.1._id: doc2 } + +--- +"pinned retriever with rank window size": + - skip: + features: headers + + - do: + headers: + # Force JSON content type so that we use a parser that interprets the floating-point score as a double + Content-Type: application/json + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: + match_all: {} + rank_window_size: 1 + + - match: { hits.total.value: 5 } + - length: { hits.hits: 1 } + - match: { hits.hits.0._id: doc1 } + - is_true: { hits.hits.0._score > 1.0 } + + - do: + headers: + # Force JSON content type so that we use a parser that interprets the floating-point score as a double + Content-Type: application/json + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: + match_all: {} + rank_window_size: 10 + + - match: { hits.total.value: 5 } + - length: { hits.hits: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.1._id: doc2 } + - is_true: { hits.hits.0._score > 1.0 } + - is_true: { hits.hits.1._score > 1.0 } + - is_true: { hits.hits.2._score <= 1.0 } + - is_true: { hits.hits.3._score <= 1.0 } + - is_true: { hits.hits.4._score <= 1.0 } + +--- +"pinned retriever explanation": + + - do: + headers: + # Force JSON content type so that we use a parser that interprets the floating-point score as a double + Content-Type: application/json + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: + match_all: {} + explain: true + + - match: { hits.hits.0._id: doc1 } + - is_true: { hits.hits.0._explanation.value > 1.0 } + - is_true: { hits.hits.0._explanation.description: "pinned documents" } + - is_true: { hits.hits.0._explanation.description: "rank [1]" } + - is_true: { hits.hits.0._explanation.details.0.details.0.details.0.details.0.details.0.description: "ConstantScore" } + +--- +"pinned retriever with empty parameters": + + - do: + search: + index: test-index1 + body: + retriever: + pinned: + retriever: + standard: + query: + match_all: {} + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.2._id: doc3 } + - match: { hits.hits.3._id: doc4 } + - match: { hits.hits.4._id: doc5 } + +--- +"pinned retriever error cases": + + - do: + catch: /\[pinned\] duplicate id found in list: doc1/ + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc1"] + retriever: + standard: + query: + match_all: {} + + - do: + catch: /\[pinned\] Max of 100 ids exceeded: 101 provided./ + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1"] * 101 + retriever: + standard: + query: + match_all: {} \ No newline at end of file From 760415dbb308bbb949f73f90f2aabeac8e333b0e Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 7 Apr 2025 17:28:55 +0100 Subject: [PATCH 06/83] Fixed merge issues --- .../retriever/PinnedRetrieverBuilderTests.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java index 53d1edfcabba8..ee952c8fdfa50 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -47,14 +47,6 @@ public static PinnedRetrieverBuilder createRandomPinnedRetrieverBuilder() { : new ArrayList<>(); return new PinnedRetrieverBuilder(ids, docs, TestRetrieverBuilder.createRandomTestRetrieverBuilder(), randomIntBetween(1, 100)); - List docs = useDocs - ? List.of( - new SpecifiedDocument(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)), - new SpecifiedDocument(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)) - ) - : new ArrayList<>(); - - return new PinnedRetrieverBuilder(ids, docs, TestRetrieverBuilder.createRandomTestRetrieverBuilder(), randomIntBetween(1, 100)); } @Override @@ -67,7 +59,6 @@ protected PinnedRetrieverBuilder doParseInstance(XContentParser parser) throws I return (PinnedRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( parser, new RetrieverParserContext(new SearchUsage(), nf -> true) - new RetrieverParserContext(new SearchUsage(), nf -> true) ); } @@ -79,7 +70,6 @@ protected boolean supportsUnknownFields() { @Override protected String[] getShuffleFieldsExceptions() { return new String[] { PinnedRetrieverBuilder.IDS_FIELD.getPreferredName(), PinnedRetrieverBuilder.DOCS_FIELD.getPreferredName() }; - return new String[] { PinnedRetrieverBuilder.IDS_FIELD.getPreferredName(), PinnedRetrieverBuilder.DOCS_FIELD.getPreferredName() }; } @Override @@ -169,5 +159,3 @@ public void testPinnedRetrieverParsing() throws IOException { } } } - -} From eec5f603e0b164b649d64a1c97e26511ec752899 Mon Sep 17 00:00:00 2001 From: Mridula Date: Tue, 8 Apr 2025 11:10:17 +0100 Subject: [PATCH 07/83] Added cluster test feature to the integration tests --- .../test/searchbusinessrules/10_pinned_retriever.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml index c10649b1e10e5..a1c25ad5286b5 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml @@ -53,6 +53,9 @@ setup: --- "pinned retriever parameter variations": + - requires: + cluster_features: ["gte_v7.15.0"] + reason: "docs parameter support was added in 7.15" - do: search: @@ -108,6 +111,9 @@ setup: --- "pinned retriever with combined parameters": + - requires: + cluster_features: ["gte_v7.15.0"] + reason: "combined parameters support was added in 7.15" - do: search: @@ -265,7 +271,6 @@ setup: - do: headers: - # Force JSON content type so that we use a parser that interprets the floating-point score as a double Content-Type: application/json search: index: test-index1 @@ -286,7 +291,6 @@ setup: - do: headers: - # Force JSON content type so that we use a parser that interprets the floating-point score as a double Content-Type: application/json search: index: test-index1 @@ -315,7 +319,6 @@ setup: - do: headers: - # Force JSON content type so that we use a parser that interprets the floating-point score as a double Content-Type: application/json search: index: test-index1 From 69e14cc43c3858d9b2f18a0e1ee84948e314ae03 Mon Sep 17 00:00:00 2001 From: Mridula Date: Tue, 8 Apr 2025 11:43:01 +0100 Subject: [PATCH 08/83] Update docs/changelog/126401.yaml --- docs/changelog/126401.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/126401.yaml diff --git a/docs/changelog/126401.yaml b/docs/changelog/126401.yaml new file mode 100644 index 0000000000000..1ad3cc4ef7f12 --- /dev/null +++ b/docs/changelog/126401.yaml @@ -0,0 +1,5 @@ +pr: 126401 +summary: Pinned retriever +area: Relevance +type: enhancement +issues: [] From 626ebac00835ba30625e491c9ab4e4316108ae52 Mon Sep 17 00:00:00 2001 From: Mridula Date: Tue, 8 Apr 2025 11:31:19 +0100 Subject: [PATCH 09/83] Registered retriever plugin --- .../xpack/searchbusinessrules/SearchBusinessRules.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java index fcbb937e4b2a0..609e50ec49f91 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -9,6 +9,8 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder; import java.util.List; @@ -21,4 +23,8 @@ public List> getQueries() { return singletonList(new QuerySpec<>(PinnedQueryBuilder.NAME, PinnedQueryBuilder::new, PinnedQueryBuilder::fromXContent)); } + @Override + public List> getRetrievers() { + return singletonList(new RetrieverSpec<>(new ParseField(PinnedRetrieverBuilder.NAME), PinnedRetrieverBuilder::fromXContent)); + } } From ef5f50f2b9428bfcfba8cb9cb131e6b30f8fc2db Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 9 Apr 2025 13:52:52 +0100 Subject: [PATCH 10/83] Enhanced changelog description --- docs/changelog/126401.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/126401.yaml b/docs/changelog/126401.yaml index 1ad3cc4ef7f12..227a6c5a61795 100644 --- a/docs/changelog/126401.yaml +++ b/docs/changelog/126401.yaml @@ -1,5 +1,5 @@ pr: 126401 -summary: Pinned retriever +summary: Add pinned retriever area: Relevance type: enhancement issues: [] From 721b1d1fde6fd171376d5b94fbf29e8564f1ca59 Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 9 Apr 2025 14:08:49 +0100 Subject: [PATCH 11/83] Added validation to the constructor creation --- .../retriever/PinnedRetrieverBuilder.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index 753d9c4e97b92..896ec41ef2dec 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -79,8 +79,15 @@ public static PinnedRetrieverBuilder fromXContent(XContentParser parser, Retriev private final List ids; private final List docs; + private void validateIdsAndDocs(List ids, List docs) { + if (ids != null && docs != null && ids.isEmpty() == false && docs.isEmpty() == false) { + throw new IllegalArgumentException("Both 'ids' and 'docs' cannot be specified at the same time"); + } + } + public PinnedRetrieverBuilder(List ids, List docs, RetrieverBuilder retrieverBuilder, int rankWindowSize) { super(new ArrayList<>(), rankWindowSize); + validateIdsAndDocs(ids, docs); this.ids = ids != null ? ids : new ArrayList<>(); this.docs = docs != null ? docs : new ArrayList<>(); addChild(new PinnedRetrieverBuilderWrapper(retrieverBuilder)); @@ -95,6 +102,7 @@ public PinnedRetrieverBuilder( List preFilterQueryBuilders ) { super(retrieverSource, rankWindowSize); + validateIdsAndDocs(ids, docs); this.ids = ids; this.docs = docs; this.retrieverName = retrieverName; From a75926c9a1a9392bf34002f06874e0b688c1d7c2 Mon Sep 17 00:00:00 2001 From: Mridula Date: Thu, 10 Apr 2025 11:16:06 +0100 Subject: [PATCH 12/83] Introduced validations in the code and incorporated in tests - compiling version --- .../retriever/PinnedRankDoc.java | 27 ++++++ .../retriever/PinnedRetrieverBuilder.java | 15 ++- .../PinnedRetrieverBuilderTests.java | 93 ++++++++++++++++--- 3 files changed, 118 insertions(+), 17 deletions(-) create mode 100644 x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java new file mode 100644 index 0000000000000..45f989d06c606 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java @@ -0,0 +1,27 @@ +package org.elasticsearch.xpack.searchbusinessrules.retriever; + +import org.elasticsearch.search.rank.RankDoc; + +public class PinnedRankDoc extends RankDoc { + private final boolean isPinned; + private final String pinnedBy; + + public PinnedRankDoc(int docId, float score, int shardIndex, boolean isPinned, String pinnedBy) { + super(docId, score, shardIndex); + this.isPinned = isPinned; + this.pinnedBy = pinnedBy; + } + + public boolean isPinned() { + return isPinned; + } + + public String getPinnedBy() { + return pinnedBy; + } + + @Override + public String toString() { + return super.toString() + ", isPinned=" + isPinned + ", pinnedBy=" + pinnedBy; + } +} \ No newline at end of file diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index 896ec41ef2dec..ac2a000064b04 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -85,6 +85,12 @@ private void validateIdsAndDocs(List ids, List docs) } } + private void validateSort(SearchSourceBuilder source) { + if (source.sorts() != null && source.sorts().isEmpty() == false) { + throw new IllegalArgumentException("Pinned retriever only supports sorting by score. Custom sorting is not allowed."); + } + } + public PinnedRetrieverBuilder(List ids, List docs, RetrieverBuilder retrieverBuilder, int rankWindowSize) { super(new ArrayList<>(), rankWindowSize); validateIdsAndDocs(ids, docs); @@ -137,6 +143,7 @@ private QueryBuilder createPinnedQuery(QueryBuilder baseQuery) { @Override protected SearchSourceBuilder finalizeSourceBuilder(SearchSourceBuilder source) { + validateSort(source); source.query(createPinnedQuery(source.query())); return source; } @@ -169,7 +176,10 @@ protected RankDoc[] combineInnerRetrieverResults(List rankResults, b RankDoc[] rankDocs = new RankDoc[scoreDocs.length]; for (int i = 0; i < scoreDocs.length; i++) { ScoreDoc scoreDoc = scoreDocs[i]; - rankDocs[i] = new RankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex); + boolean isPinned = docs.stream().anyMatch(doc -> doc.id().equals(String.valueOf(scoreDoc.doc))) || + ids.contains(String.valueOf(scoreDoc.doc)); + String pinnedBy = docs.stream().anyMatch(doc -> doc.id().equals(String.valueOf(scoreDoc.doc))) ? "docs" : "ids"; + rankDocs[i] = new PinnedRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex, isPinned, pinnedBy); rankDocs[i].rank = i + 1; } return rankDocs; @@ -207,7 +217,8 @@ public QueryBuilder topDocsQuery() { @Override public QueryBuilder explainQuery() { - return createPinnedQuery(in.explainQuery()); + QueryBuilder baseQuery = in.explainQuery(); + return createPinnedQuery(baseQuery); } } } diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java index ee952c8fdfa50..d166e37d24895 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -27,28 +27,16 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.stream.IntStream; +import java.util.stream.Collectors; import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.containsString; public class PinnedRetrieverBuilderTests extends AbstractXContentTestCase { - public static PinnedRetrieverBuilder createRandomPinnedRetrieverBuilder() { - boolean useIds = randomBoolean(); - boolean useDocs = useIds == false || randomBoolean(); - - List ids = useIds ? List.of(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)) : new ArrayList<>(); - List docs = useDocs - ? List.of( - new SpecifiedDocument(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)), - new SpecifiedDocument(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)) - ) - : new ArrayList<>(); - - return new PinnedRetrieverBuilder(ids, docs, TestRetrieverBuilder.createRandomTestRetrieverBuilder(), randomIntBetween(1, 100)); - } - @Override protected PinnedRetrieverBuilder createTestInstance() { return createRandomPinnedRetrieverBuilder(); @@ -62,6 +50,26 @@ protected PinnedRetrieverBuilder doParseInstance(XContentParser parser) throws I ); } + public static PinnedRetrieverBuilder createRandomPinnedRetrieverBuilder() { + boolean useIds = randomBoolean(); + int numItems = randomIntBetween(1, 5); + + List ids = useIds ? + IntStream.range(0, numItems) + .mapToObj(i -> randomAlphaOfLengthBetween(5, 10)) + .collect(Collectors.toList()) : + new ArrayList<>(); + List docs = useIds ? + new ArrayList<>() : + IntStream.range(0, numItems) + .mapToObj(i -> new SpecifiedDocument( + randomAlphaOfLengthBetween(5, 10), + randomAlphaOfLengthBetween(5, 10) + )) + .collect(Collectors.toList()); + return new PinnedRetrieverBuilder(ids, docs, TestRetrieverBuilder.createRandomTestRetrieverBuilder(), randomIntBetween(1, 100)); + } + @Override protected boolean supportsUnknownFields() { return false; @@ -93,6 +101,61 @@ protected NamedXContentRegistry xContentRegistry() { return new NamedXContentRegistry(entries); } + public void testValidation() { + List ids = List.of("id1", "id2"); + List docs = List.of( + new SpecifiedDocument("id3", "index3"), + new SpecifiedDocument("id4", "index4") + ); + + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new PinnedRetrieverBuilder(ids, docs, TestRetrieverBuilder.createRandomTestRetrieverBuilder(), 10) + ); + assertThat(e.getMessage(), equalTo("Both 'ids' and 'docs' cannot be specified at the same time")); + + e = expectThrows( + IllegalArgumentException.class, + () -> new PinnedRetrieverBuilder(new ArrayList<>(), new ArrayList<>(), TestRetrieverBuilder.createRandomTestRetrieverBuilder(), 10) + ); + assertThat(e.getMessage(), equalTo("Either 'ids' or 'docs' must be specified")); + + assertNotNull(new PinnedRetrieverBuilder(ids, new ArrayList<>(), TestRetrieverBuilder.createRandomTestRetrieverBuilder(), 10)); + assertNotNull(new PinnedRetrieverBuilder(new ArrayList<>(), docs, TestRetrieverBuilder.createRandomTestRetrieverBuilder(), 10)); + } + + public void testValidateSort() { + PinnedRetrieverBuilder builder = createRandomPinnedRetrieverBuilder(); + + // Test empty sort is allowed + final SearchSourceBuilder emptySource = new SearchSourceBuilder(); + builder.finalizeSourceBuilder(emptySource); + + // Test score sort is allowed + final SearchSourceBuilder scoreSource = new SearchSourceBuilder(); + scoreSource.sort("_score"); + builder.finalizeSourceBuilder(scoreSource); + + // Test custom sort is not allowed + final SearchSourceBuilder customSortSource = new SearchSourceBuilder(); + customSortSource.sort("field"); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> builder.finalizeSourceBuilder(customSortSource) + ); + assertThat(e.getMessage(), equalTo("Pinned retriever only supports sorting by score. Custom sorting is not allowed.")); + + // Test multiple sorts including custom sort is not allowed + final SearchSourceBuilder multipleSortsSource = new SearchSourceBuilder(); + multipleSortsSource.sort("_score"); + multipleSortsSource.sort("field"); + e = expectThrows( + IllegalArgumentException.class, + () -> builder.finalizeSourceBuilder(multipleSortsSource) + ); + assertThat(e.getMessage(), equalTo("Pinned retriever only supports sorting by score. Custom sorting is not allowed.")); + } + public void testParserDefaults() throws IOException { // Inner retriever content only sent to parser String json = """ From c84028ec395d01075d1953193d4d11a2d61627f9 Mon Sep 17 00:00:00 2001 From: Mridula Date: Thu, 10 Apr 2025 13:34:52 +0100 Subject: [PATCH 13/83] Included PinnedRankDoc to enhance the explain query, validations for inputs anfd sorting. Fixed the tests after these changes --- .../retriever/PinnedRankDoc.java | 2 +- .../retriever/PinnedRetrieverBuilder.java | 15 +- .../PinnedRetrieverBuilderTests.java | 135 ++++++++---------- 3 files changed, 68 insertions(+), 84 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java index 45f989d06c606..f3eb2769e99b4 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java @@ -24,4 +24,4 @@ public String getPinnedBy() { public String toString() { return super.toString() + ", isPinned=" + isPinned + ", pinnedBy=" + pinnedBy; } -} \ No newline at end of file +} diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index ac2a000064b04..a8ba331787f75 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -9,6 +9,7 @@ import org.apache.lucene.search.ScoreDoc; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.rank.RankDoc; @@ -16,6 +17,8 @@ import org.elasticsearch.search.retriever.RetrieverBuilder; import org.elasticsearch.search.retriever.RetrieverBuilderWrapper; import org.elasticsearch.search.retriever.RetrieverParserContext; +import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentBuilder; @@ -80,13 +83,14 @@ public static PinnedRetrieverBuilder fromXContent(XContentParser parser, Retriev private final List docs; private void validateIdsAndDocs(List ids, List docs) { - if (ids != null && docs != null && ids.isEmpty() == false && docs.isEmpty() == false) { + if ((ids != null && ids.isEmpty() == false) && (docs != null && docs.isEmpty() == false)) { throw new IllegalArgumentException("Both 'ids' and 'docs' cannot be specified at the same time"); } } private void validateSort(SearchSourceBuilder source) { - if (source.sorts() != null && source.sorts().isEmpty() == false) { + List> sorts = source.sorts(); + if (sorts != null && sorts.stream().anyMatch(sort -> sort instanceof ScoreSortBuilder == false)) { throw new IllegalArgumentException("Pinned retriever only supports sorting by score. Custom sorting is not allowed."); } } @@ -132,6 +136,9 @@ public int rankWindowSize() { * @return a PinnedQueryBuilder or the original query if no pinned documents */ private QueryBuilder createPinnedQuery(QueryBuilder baseQuery) { + if (baseQuery == null) { + baseQuery = new MatchAllQueryBuilder(); + } if (docs.isEmpty() == false) { return new PinnedQueryBuilder(baseQuery, docs.toArray(new SpecifiedDocument[0])); } else if (ids.isEmpty() == false) { @@ -176,8 +183,8 @@ protected RankDoc[] combineInnerRetrieverResults(List rankResults, b RankDoc[] rankDocs = new RankDoc[scoreDocs.length]; for (int i = 0; i < scoreDocs.length; i++) { ScoreDoc scoreDoc = scoreDocs[i]; - boolean isPinned = docs.stream().anyMatch(doc -> doc.id().equals(String.valueOf(scoreDoc.doc))) || - ids.contains(String.valueOf(scoreDoc.doc)); + boolean isPinned = docs.stream().anyMatch(doc -> doc.id().equals(String.valueOf(scoreDoc.doc))) + || ids.contains(String.valueOf(scoreDoc.doc)); String pinnedBy = docs.stream().anyMatch(doc -> doc.id().equals(String.valueOf(scoreDoc.doc))) ? "docs" : "ids"; rankDocs[i] = new PinnedRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex, isPinned, pinnedBy); rankDocs[i].rank = i + 1; diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java index d166e37d24895..b74508910b319 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -27,13 +27,12 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.stream.IntStream; import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.containsString; public class PinnedRetrieverBuilderTests extends AbstractXContentTestCase { @@ -54,18 +53,13 @@ public static PinnedRetrieverBuilder createRandomPinnedRetrieverBuilder() { boolean useIds = randomBoolean(); int numItems = randomIntBetween(1, 5); - List ids = useIds ? - IntStream.range(0, numItems) - .mapToObj(i -> randomAlphaOfLengthBetween(5, 10)) - .collect(Collectors.toList()) : - new ArrayList<>(); - List docs = useIds ? - new ArrayList<>() : - IntStream.range(0, numItems) - .mapToObj(i -> new SpecifiedDocument( - randomAlphaOfLengthBetween(5, 10), - randomAlphaOfLengthBetween(5, 10) - )) + List ids = useIds + ? IntStream.range(0, numItems).mapToObj(i -> randomAlphaOfLengthBetween(5, 10)).collect(Collectors.toList()) + : new ArrayList<>(); + List docs = useIds + ? new ArrayList<>() + : IntStream.range(0, numItems) + .mapToObj(i -> new SpecifiedDocument(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10))) .collect(Collectors.toList()); return new PinnedRetrieverBuilder(ids, docs, TestRetrieverBuilder.createRandomTestRetrieverBuilder(), randomIntBetween(1, 100)); } @@ -101,61 +95,6 @@ protected NamedXContentRegistry xContentRegistry() { return new NamedXContentRegistry(entries); } - public void testValidation() { - List ids = List.of("id1", "id2"); - List docs = List.of( - new SpecifiedDocument("id3", "index3"), - new SpecifiedDocument("id4", "index4") - ); - - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> new PinnedRetrieverBuilder(ids, docs, TestRetrieverBuilder.createRandomTestRetrieverBuilder(), 10) - ); - assertThat(e.getMessage(), equalTo("Both 'ids' and 'docs' cannot be specified at the same time")); - - e = expectThrows( - IllegalArgumentException.class, - () -> new PinnedRetrieverBuilder(new ArrayList<>(), new ArrayList<>(), TestRetrieverBuilder.createRandomTestRetrieverBuilder(), 10) - ); - assertThat(e.getMessage(), equalTo("Either 'ids' or 'docs' must be specified")); - - assertNotNull(new PinnedRetrieverBuilder(ids, new ArrayList<>(), TestRetrieverBuilder.createRandomTestRetrieverBuilder(), 10)); - assertNotNull(new PinnedRetrieverBuilder(new ArrayList<>(), docs, TestRetrieverBuilder.createRandomTestRetrieverBuilder(), 10)); - } - - public void testValidateSort() { - PinnedRetrieverBuilder builder = createRandomPinnedRetrieverBuilder(); - - // Test empty sort is allowed - final SearchSourceBuilder emptySource = new SearchSourceBuilder(); - builder.finalizeSourceBuilder(emptySource); - - // Test score sort is allowed - final SearchSourceBuilder scoreSource = new SearchSourceBuilder(); - scoreSource.sort("_score"); - builder.finalizeSourceBuilder(scoreSource); - - // Test custom sort is not allowed - final SearchSourceBuilder customSortSource = new SearchSourceBuilder(); - customSortSource.sort("field"); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> builder.finalizeSourceBuilder(customSortSource) - ); - assertThat(e.getMessage(), equalTo("Pinned retriever only supports sorting by score. Custom sorting is not allowed.")); - - // Test multiple sorts including custom sort is not allowed - final SearchSourceBuilder multipleSortsSource = new SearchSourceBuilder(); - multipleSortsSource.sort("_score"); - multipleSortsSource.sort("field"); - e = expectThrows( - IllegalArgumentException.class, - () -> builder.finalizeSourceBuilder(multipleSortsSource) - ); - assertThat(e.getMessage(), equalTo("Pinned retriever only supports sorting by score. Custom sorting is not allowed.")); - } - public void testParserDefaults() throws IOException { // Inner retriever content only sent to parser String json = """ @@ -187,16 +126,6 @@ public void testPinnedRetrieverParsing() throws IOException { "id1", "id2" ], - "docs": [ - { - "_index": "index1", - "_id": "doc1" - }, - { - "_index": "index2", - "_id": "doc2" - } - ], "rank_window_size": 100, "_name": "my_pinned_retriever" } @@ -221,4 +150,52 @@ public void testPinnedRetrieverParsing() throws IOException { } } } + + public void testValidation() { + expectThrows(IllegalArgumentException.class, () -> { + new PinnedRetrieverBuilder( + List.of("id1"), + List.of(new SpecifiedDocument("id2", "index")), + new TestRetrieverBuilder("test"), + DEFAULT_RANK_WINDOW_SIZE + ); + }); + + PinnedRetrieverBuilder builder = new PinnedRetrieverBuilder( + List.of(), + List.of(), + new TestRetrieverBuilder("test"), + DEFAULT_RANK_WINDOW_SIZE + ); + assertNotNull(builder); + } + + public void testValidateSort() { + PinnedRetrieverBuilder builder = new PinnedRetrieverBuilder( + List.of("id1"), + List.of(), + new TestRetrieverBuilder("test"), + DEFAULT_RANK_WINDOW_SIZE + ); + + SearchSourceBuilder emptySource = new SearchSourceBuilder(); + builder.finalizeSourceBuilder(emptySource); + assertThat(emptySource.sorts(), equalTo(null)); + + SearchSourceBuilder scoreSource = new SearchSourceBuilder(); + scoreSource.sort("_score"); + builder.finalizeSourceBuilder(scoreSource); + assertThat(scoreSource.sorts().size(), equalTo(1)); + + SearchSourceBuilder customSortSource = new SearchSourceBuilder(); + customSortSource.sort("field1"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.finalizeSourceBuilder(customSortSource)); + assertThat(e.getMessage(), equalTo("Pinned retriever only supports sorting by score. Custom sorting is not allowed.")); + + SearchSourceBuilder multipleSortsSource = new SearchSourceBuilder(); + multipleSortsSource.sort("_score"); + multipleSortsSource.sort("field1"); + e = expectThrows(IllegalArgumentException.class, () -> builder.finalizeSourceBuilder(multipleSortsSource)); + assertThat(e.getMessage(), equalTo("Pinned retriever only supports sorting by score. Custom sorting is not allowed.")); + } } From 447656430fba7b1dc04928999be513d3dfebeea4 Mon Sep 17 00:00:00 2001 From: Mridula Date: Thu, 10 Apr 2025 19:35:02 +0100 Subject: [PATCH 14/83] Working on improving integration test and introducing cluster features --- .../plugin/search-business-rules/build.gradle | 17 +++- .../src/main/java/module-info.java | 2 + .../SearchBusinessRulesFeatures.java | 27 ++++++ .../retriever/PinnedRankDoc.java | 7 ++ ...lasticsearch.features.FeatureSpecification | 8 ++ .../10_pinned_retriever.yml | 83 ++++++++++++++----- 6 files changed, 121 insertions(+), 23 deletions(-) create mode 100644 x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java create mode 100644 x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification diff --git a/x-pack/plugin/search-business-rules/build.gradle b/x-pack/plugin/search-business-rules/build.gradle index cce6e15aedc06..3fb27ede9addf 100644 --- a/x-pack/plugin/search-business-rules/build.gradle +++ b/x-pack/plugin/search-business-rules/build.gradle @@ -1,5 +1,6 @@ apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' esplugin { name = 'search-business-rules' @@ -7,12 +8,22 @@ esplugin { classname ='org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRules' extendedPlugins = ['x-pack-core'] } -base { - archivesName = 'x-pack-searchbusinessrules' -} dependencies { compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(":test:framework") + + yamlRestTestImplementation(testArtifact(project(xpackModule('core')))) +} + +restResources { + restTests { + includeXpack 'searchbusinessrules' + } +} + +testClusters.matching { it.name == "yamlRestTest" }.configureEach { + testDistribution = 'DEFAULT' + setting 'xpack.license.self_generated.type', 'trial' } diff --git a/x-pack/plugin/search-business-rules/src/main/java/module-info.java b/x-pack/plugin/search-business-rules/src/main/java/module-info.java index d11c809b43854..d83d909644a5a 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/module-info.java +++ b/x-pack/plugin/search-business-rules/src/main/java/module-info.java @@ -16,4 +16,6 @@ requires org.elasticsearch.xcore; exports org.elasticsearch.xpack.searchbusinessrules; + + provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRulesFeatures; } diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java new file mode 100644 index 0000000000000..24a7282559caf --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.searchbusinessrules; + +import org.elasticsearch.features.FeatureSpecification; +import org.elasticsearch.features.NodeFeature; + +import java.util.Set; + +public class SearchBusinessRulesFeatures implements FeatureSpecification { + public static final NodeFeature PINNED_RETRIEVER_TEST_FEATURE = new NodeFeature("pinned_retriever_test"); + + @Override + public Set getFeatures() { + return Set.of(PINNED_RETRIEVER_TEST_FEATURE); + } + + @Override + public Set getTestFeatures() { + return Set.of(PINNED_RETRIEVER_TEST_FEATURE); + } +} diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java index f3eb2769e99b4..9f4dcbaa6cc78 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java @@ -1,3 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + package org.elasticsearch.xpack.searchbusinessrules.retriever; import org.elasticsearch.search.rank.RankDoc; diff --git a/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification new file mode 100644 index 0000000000000..d0994d77c29c5 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -0,0 +1,8 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRulesFeatures \ No newline at end of file diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml index a1c25ad5286b5..86394b3857f9d 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml @@ -1,4 +1,8 @@ setup: + - requires: + cluster_features: ["pinned_retriever_test"] + reason: "pinned retriever test feature is required" + - do: indices.create: index: test-index1 @@ -31,7 +35,6 @@ setup: --- "pinned retriever basic functionality": - - do: search: index: test-index1 @@ -50,13 +53,14 @@ setup: - match: { hits.hits.2._id: doc3 } - match: { hits.hits.3._id: doc4 } - match: { hits.hits.4._id: doc5 } + - is_true: { hits.hits.0._score > 1.0 } + - is_true: { hits.hits.1._score > 1.0 } + - is_true: { hits.hits.2._score <= 1.0 } + - is_true: { hits.hits.3._score <= 1.0 } + - is_true: { hits.hits.4._score <= 1.0 } --- "pinned retriever parameter variations": - - requires: - cluster_features: ["gte_v7.15.0"] - reason: "docs parameter support was added in 7.15" - - do: search: index: test-index1 @@ -76,6 +80,11 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } + - is_true: { hits.hits.0._score > 1.0 } + - is_true: { hits.hits.1._score > 1.0 } + - is_true: { hits.hits.2._score <= 1.0 } + - is_true: { hits.hits.3._score <= 1.0 } + - is_true: { hits.hits.4._score <= 1.0 } - do: search: @@ -91,6 +100,11 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } + - is_true: { hits.hits.0._score > 1.0 } + - is_true: { hits.hits.1._score <= 1.0 } + - is_true: { hits.hits.2._score <= 1.0 } + - is_true: { hits.hits.3._score <= 1.0 } + - is_true: { hits.hits.4._score <= 1.0 } - do: search: @@ -108,13 +122,14 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } + - is_true: { hits.hits.0._score > 1.0 } + - is_true: { hits.hits.1._score <= 1.0 } + - is_true: { hits.hits.2._score <= 1.0 } + - is_true: { hits.hits.3._score <= 1.0 } + - is_true: { hits.hits.4._score <= 1.0 } --- "pinned retriever with combined parameters": - - requires: - cluster_features: ["gte_v7.15.0"] - reason: "combined parameters support was added in 7.15" - - do: search: index: test-index1 @@ -133,6 +148,11 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } + - is_true: { hits.hits.0._score > 1.0 } + - is_true: { hits.hits.1._score > 1.0 } + - is_true: { hits.hits.2._score <= 1.0 } + - is_true: { hits.hits.3._score <= 1.0 } + - is_true: { hits.hits.4._score <= 1.0 } - do: search: @@ -156,10 +176,14 @@ setup: - match: { hits.hits.1._id: doc3 } - match: { hits.hits.2._id: doc2 } - match: { hits.hits.3._id: doc4 } + - is_true: { hits.hits.0._score > 1.0 } + - is_true: { hits.hits.1._score > 1.0 } + - is_true: { hits.hits.2._score > 1.0 } + - is_true: { hits.hits.3._score > 1.0 } + - is_true: { hits.hits.4._score <= 1.0 } --- "pinned retriever combined with rrf": - - do: search: index: test-index1 @@ -173,14 +197,18 @@ setup: { standard: { query: { - match_all: {} + match: { + text: "document one" + } } } }, { standard: { query: { - match_all: {} + match: { + text: "document two" + } } } } @@ -189,10 +217,13 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } + - is_true: { hits.hits.0._score > 1.0 } + - is_true: { hits.hits.1._score > 1.0 } + - is_true: { hits.hits.2._score > hits.hits.3._score } + - is_true: { hits.hits.3._score > hits.hits.4._score } --- "pinned retriever with pagination": - - do: search: index: test-index1 @@ -210,10 +241,10 @@ setup: - match: { hits.total.value: 5 } - length: { hits.hits: 1 } - match: { hits.hits.0._id: doc2 } + - is_true: { hits.hits.0._score > 1.0 } --- "pinned retriever as a sub-retriever": - - do: search: index: test-index1 @@ -243,10 +274,14 @@ setup: ] - match: { hits.total.value: 5 } + - is_true: { hits.hits.0._score > 1.0 } + - is_true: { hits.hits.1._score > 1.0 } + - is_true: { hits.hits.2._score <= 1.0 } + - is_true: { hits.hits.3._score <= 1.0 } + - is_true: { hits.hits.4._score <= 1.0 } --- "pinned retriever with explicit sort on score": - - do: search: index: test-index1 @@ -263,6 +298,11 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } + - is_true: { hits.hits.0._score > 1.0 } + - is_true: { hits.hits.1._score > 1.0 } + - is_true: { hits.hits.2._score <= 1.0 } + - is_true: { hits.hits.3._score <= 1.0 } + - is_true: { hits.hits.4._score <= 1.0 } --- "pinned retriever with rank window size": @@ -316,7 +356,6 @@ setup: --- "pinned retriever explanation": - - do: headers: Content-Type: application/json @@ -335,12 +374,17 @@ setup: - match: { hits.hits.0._id: doc1 } - is_true: { hits.hits.0._explanation.value > 1.0 } - is_true: { hits.hits.0._explanation.description: "pinned documents" } - - is_true: { hits.hits.0._explanation.description: "rank [1]" } - - is_true: { hits.hits.0._explanation.details.0.details.0.details.0.details.0.details.0.description: "ConstantScore" } + - is_true: { hits.hits.0._explanation.details.0.details.0.details.0.description: "pinned_by: [ids]" } + - is_true: { hits.hits.0._explanation.details.0.details.0.details.1.description: "is_pinned: true" } + + - match: { hits.hits.1._id: doc2 } + - is_true: { hits.hits.1._explanation.value > 1.0 } + - is_true: { hits.hits.1._explanation.description: "pinned documents" } + - is_true: { hits.hits.1._explanation.details.0.details.0.details.0.description: "pinned_by: [ids]" } + - is_true: { hits.hits.1._explanation.details.0.details.0.details.1.description: "is_pinned: true" } --- "pinned retriever with empty parameters": - - do: search: index: test-index1 @@ -361,7 +405,6 @@ setup: --- "pinned retriever error cases": - - do: catch: /\[pinned\] duplicate id found in list: doc1/ search: From eaee938edd3e048e42860ff65589d4da7b7ba81e Mon Sep 17 00:00:00 2001 From: Mridula Date: Thu, 10 Apr 2025 22:27:11 +0100 Subject: [PATCH 15/83] Removed cluster test temporarily --- .../plugin/search-business-rules/build.gradle | 16 +++-------- .../src/main/java/module-info.java | 2 -- .../SearchBusinessRules.java | 4 --- .../SearchBusinessRulesFeatures.java | 27 ------------------- .../10_pinned_retriever.yml | 4 --- 5 files changed, 3 insertions(+), 50 deletions(-) delete mode 100644 x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java diff --git a/x-pack/plugin/search-business-rules/build.gradle b/x-pack/plugin/search-business-rules/build.gradle index 3fb27ede9addf..580b6fcebe2fc 100644 --- a/x-pack/plugin/search-business-rules/build.gradle +++ b/x-pack/plugin/search-business-rules/build.gradle @@ -8,22 +8,12 @@ esplugin { classname ='org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRules' extendedPlugins = ['x-pack-core'] } +base { + archivesName = 'x-pack-searchbusinessrules' +} dependencies { compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(":test:framework") - - yamlRestTestImplementation(testArtifact(project(xpackModule('core')))) -} - -restResources { - restTests { - includeXpack 'searchbusinessrules' - } -} - -testClusters.matching { it.name == "yamlRestTest" }.configureEach { - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'trial' } diff --git a/x-pack/plugin/search-business-rules/src/main/java/module-info.java b/x-pack/plugin/search-business-rules/src/main/java/module-info.java index d83d909644a5a..d11c809b43854 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/module-info.java +++ b/x-pack/plugin/search-business-rules/src/main/java/module-info.java @@ -16,6 +16,4 @@ requires org.elasticsearch.xcore; exports org.elasticsearch.xpack.searchbusinessrules; - - provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRulesFeatures; } diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java index 609e50ec49f91..30070321062cc 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -23,8 +23,4 @@ public List> getQueries() { return singletonList(new QuerySpec<>(PinnedQueryBuilder.NAME, PinnedQueryBuilder::new, PinnedQueryBuilder::fromXContent)); } - @Override - public List> getRetrievers() { - return singletonList(new RetrieverSpec<>(new ParseField(PinnedRetrieverBuilder.NAME), PinnedRetrieverBuilder::fromXContent)); - } } diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java deleted file mode 100644 index 24a7282559caf..0000000000000 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.searchbusinessrules; - -import org.elasticsearch.features.FeatureSpecification; -import org.elasticsearch.features.NodeFeature; - -import java.util.Set; - -public class SearchBusinessRulesFeatures implements FeatureSpecification { - public static final NodeFeature PINNED_RETRIEVER_TEST_FEATURE = new NodeFeature("pinned_retriever_test"); - - @Override - public Set getFeatures() { - return Set.of(PINNED_RETRIEVER_TEST_FEATURE); - } - - @Override - public Set getTestFeatures() { - return Set.of(PINNED_RETRIEVER_TEST_FEATURE); - } -} diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml index 86394b3857f9d..ac3791fbc2ded 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml @@ -1,8 +1,4 @@ setup: - - requires: - cluster_features: ["pinned_retriever_test"] - reason: "pinned retriever test feature is required" - - do: indices.create: index: test-index1 From 2aad14513623a7eb5e9e15cd04b6a2bcb82feee4 Mon Sep 17 00:00:00 2001 From: Mridula Date: Thu, 10 Apr 2025 23:18:01 +0100 Subject: [PATCH 16/83] Got pinnedretriever yaml to working state without cluster --- .../plugin/search-business-rules/build.gradle | 35 ++++- ...lasticsearch.features.FeatureSpecification | 8 -- .../10_pinned_retriever.yml | 128 +++++++++--------- 3 files changed, 98 insertions(+), 73 deletions(-) delete mode 100644 x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification diff --git a/x-pack/plugin/search-business-rules/build.gradle b/x-pack/plugin/search-business-rules/build.gradle index 580b6fcebe2fc..0e021d4e7fe94 100644 --- a/x-pack/plugin/search-business-rules/build.gradle +++ b/x-pack/plugin/search-business-rules/build.gradle @@ -15,5 +15,38 @@ base { dependencies { compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) - testImplementation project(":test:framework") + testImplementation(testArtifact(project(':server'))) + + clusterModules project(':modules:mapper-extras') + clusterModules project(xpackModule('search-business-rules')) + clusterModules project(':modules:lang-painless') +} + +restResources { + restApi { + include '_common', + 'bulk', + 'cluster', + 'indices', + 'index', + 'search', + 'xpack' + } + restTests { + includeXpack 'searchbusinessrules' + } +} + +tasks.named("yamlRestTest") { + usesDefaultDistribution("uses search business rules plugin") +} + +artifacts { + restXpackTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test")) +} + +testClusters.configureEach { + testDistribution = 'DEFAULT' + setting 'xpack.license.self_generated.type', 'trial' + setting 'xpack.security.enabled', 'false' } diff --git a/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification deleted file mode 100644 index d0994d77c29c5..0000000000000 --- a/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification +++ /dev/null @@ -1,8 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0; you may not use this file except in compliance with the Elastic License -# 2.0. -# - -org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRulesFeatures \ No newline at end of file diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml index ac3791fbc2ded..8e6b106c1c21e 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml @@ -49,11 +49,11 @@ setup: - match: { hits.hits.2._id: doc3 } - match: { hits.hits.3._id: doc4 } - match: { hits.hits.4._id: doc5 } - - is_true: { hits.hits.0._score > 1.0 } - - is_true: { hits.hits.1._score > 1.0 } - - is_true: { hits.hits.2._score <= 1.0 } - - is_true: { hits.hits.3._score <= 1.0 } - - is_true: { hits.hits.4._score <= 1.0 } + - is_true: hits.hits.0._score > 1.0 + - is_true: hits.hits.1._score > 1.0 + - is_true: hits.hits.2._score <= 1.0 + - is_true: hits.hits.3._score <= 1.0 + - is_true: hits.hits.4._score <= 1.0 --- "pinned retriever parameter variations": @@ -76,11 +76,11 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } - - is_true: { hits.hits.0._score > 1.0 } - - is_true: { hits.hits.1._score > 1.0 } - - is_true: { hits.hits.2._score <= 1.0 } - - is_true: { hits.hits.3._score <= 1.0 } - - is_true: { hits.hits.4._score <= 1.0 } + - is_true: hits.hits.0._score > 1.0 + - is_true: hits.hits.1._score > 1.0 + - is_true: hits.hits.2._score <= 1.0 + - is_true: hits.hits.3._score <= 1.0 + - is_true: hits.hits.4._score <= 1.0 - do: search: @@ -96,11 +96,11 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - - is_true: { hits.hits.0._score > 1.0 } - - is_true: { hits.hits.1._score <= 1.0 } - - is_true: { hits.hits.2._score <= 1.0 } - - is_true: { hits.hits.3._score <= 1.0 } - - is_true: { hits.hits.4._score <= 1.0 } + - is_true: hits.hits.0._score > 1.0 + - is_true: hits.hits.1._score <= 1.0 + - is_true: hits.hits.2._score <= 1.0 + - is_true: hits.hits.3._score <= 1.0 + - is_true: hits.hits.4._score <= 1.0 - do: search: @@ -118,11 +118,11 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - - is_true: { hits.hits.0._score > 1.0 } - - is_true: { hits.hits.1._score <= 1.0 } - - is_true: { hits.hits.2._score <= 1.0 } - - is_true: { hits.hits.3._score <= 1.0 } - - is_true: { hits.hits.4._score <= 1.0 } + - is_true: hits.hits.0._score > 1.0 + - is_true: hits.hits.1._score <= 1.0 + - is_true: hits.hits.2._score <= 1.0 + - is_true: hits.hits.3._score <= 1.0 + - is_true: hits.hits.4._score <= 1.0 --- "pinned retriever with combined parameters": @@ -144,11 +144,11 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } - - is_true: { hits.hits.0._score > 1.0 } - - is_true: { hits.hits.1._score > 1.0 } - - is_true: { hits.hits.2._score <= 1.0 } - - is_true: { hits.hits.3._score <= 1.0 } - - is_true: { hits.hits.4._score <= 1.0 } + - is_true: hits.hits.0._score > 1.0 + - is_true: hits.hits.1._score > 1.0 + - is_true: hits.hits.2._score <= 1.0 + - is_true: hits.hits.3._score <= 1.0 + - is_true: hits.hits.4._score <= 1.0 - do: search: @@ -172,11 +172,11 @@ setup: - match: { hits.hits.1._id: doc3 } - match: { hits.hits.2._id: doc2 } - match: { hits.hits.3._id: doc4 } - - is_true: { hits.hits.0._score > 1.0 } - - is_true: { hits.hits.1._score > 1.0 } - - is_true: { hits.hits.2._score > 1.0 } - - is_true: { hits.hits.3._score > 1.0 } - - is_true: { hits.hits.4._score <= 1.0 } + - is_true: hits.hits.0._score > 1.0 + - is_true: hits.hits.1._score > 1.0 + - is_true: hits.hits.2._score > 1.0 + - is_true: hits.hits.3._score > 1.0 + - is_true: hits.hits.4._score <= 1.0 --- "pinned retriever combined with rrf": @@ -213,10 +213,10 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } - - is_true: { hits.hits.0._score > 1.0 } - - is_true: { hits.hits.1._score > 1.0 } - - is_true: { hits.hits.2._score > hits.hits.3._score } - - is_true: { hits.hits.3._score > hits.hits.4._score } + - is_true: hits.hits.0._score > 1.0 + - is_true: hits.hits.1._score > 1.0 + - is_true: hits.hits.2._score > hits.hits.3._score + - is_true: hits.hits.3._score > hits.hits.4._score --- "pinned retriever with pagination": @@ -237,7 +237,7 @@ setup: - match: { hits.total.value: 5 } - length: { hits.hits: 1 } - match: { hits.hits.0._id: doc2 } - - is_true: { hits.hits.0._score > 1.0 } + - is_true: hits.hits.0._score > 1.0 --- "pinned retriever as a sub-retriever": @@ -270,11 +270,11 @@ setup: ] - match: { hits.total.value: 5 } - - is_true: { hits.hits.0._score > 1.0 } - - is_true: { hits.hits.1._score > 1.0 } - - is_true: { hits.hits.2._score <= 1.0 } - - is_true: { hits.hits.3._score <= 1.0 } - - is_true: { hits.hits.4._score <= 1.0 } + - is_true: hits.hits.0._score > 1.0 + - is_true: hits.hits.1._score > 1.0 + - is_true: hits.hits.2._score <= 1.0 + - is_true: hits.hits.3._score <= 1.0 + - is_true: hits.hits.4._score <= 1.0 --- "pinned retriever with explicit sort on score": @@ -294,11 +294,11 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } - - is_true: { hits.hits.0._score > 1.0 } - - is_true: { hits.hits.1._score > 1.0 } - - is_true: { hits.hits.2._score <= 1.0 } - - is_true: { hits.hits.3._score <= 1.0 } - - is_true: { hits.hits.4._score <= 1.0 } + - is_true: hits.hits.0._score > 1.0 + - is_true: hits.hits.1._score > 1.0 + - is_true: hits.hits.2._score <= 1.0 + - is_true: hits.hits.3._score <= 1.0 + - is_true: hits.hits.4._score <= 1.0 --- "pinned retriever with rank window size": @@ -323,7 +323,7 @@ setup: - match: { hits.total.value: 5 } - length: { hits.hits: 1 } - match: { hits.hits.0._id: doc1 } - - is_true: { hits.hits.0._score > 1.0 } + - is_true: hits.hits.0._score > 1.0 - do: headers: @@ -344,17 +344,15 @@ setup: - length: { hits.hits: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } - - is_true: { hits.hits.0._score > 1.0 } - - is_true: { hits.hits.1._score > 1.0 } - - is_true: { hits.hits.2._score <= 1.0 } - - is_true: { hits.hits.3._score <= 1.0 } - - is_true: { hits.hits.4._score <= 1.0 } + - is_true: hits.hits.0._score > 1.0 + - is_true: hits.hits.1._score > 1.0 + - is_true: hits.hits.2._score <= 1.0 + - is_true: hits.hits.3._score <= 1.0 + - is_true: hits.hits.4._score <= 1.0 --- "pinned retriever explanation": - do: - headers: - Content-Type: application/json search: index: test-index1 body: @@ -368,16 +366,16 @@ setup: explain: true - match: { hits.hits.0._id: doc1 } - - is_true: { hits.hits.0._explanation.value > 1.0 } - - is_true: { hits.hits.0._explanation.description: "pinned documents" } - - is_true: { hits.hits.0._explanation.details.0.details.0.details.0.description: "pinned_by: [ids]" } - - is_true: { hits.hits.0._explanation.details.0.details.0.details.1.description: "is_pinned: true" } + - is_true: hits.hits.0._explanation.value > 1.0 + - match: { hits.hits.0._explanation.description: "pinned documents" } + - match: { hits.hits.0._explanation.details.0.details.0.details.0.description: "pinned_by: [ids]" } + - match: { hits.hits.0._explanation.details.0.details.0.details.1.description: "is_pinned: true" } - match: { hits.hits.1._id: doc2 } - - is_true: { hits.hits.1._explanation.value > 1.0 } - - is_true: { hits.hits.1._explanation.description: "pinned documents" } - - is_true: { hits.hits.1._explanation.details.0.details.0.details.0.description: "pinned_by: [ids]" } - - is_true: { hits.hits.1._explanation.details.0.details.0.details.1.description: "is_pinned: true" } + - is_true: hits.hits.1._explanation.value > 1.0 + - match: { hits.hits.1._explanation.description: "pinned documents" } + - match: { hits.hits.1._explanation.details.0.details.0.details.0.description: "pinned_by: [ids]" } + - match: { hits.hits.1._explanation.details.0.details.0.details.1.description: "is_pinned: true" } --- "pinned retriever with empty parameters": @@ -415,14 +413,16 @@ setup: match_all: {} - do: - catch: /\[pinned\] Max of 100 ids exceeded: 101 provided./ + catch: /\[pinned\] Cannot specify both ids and docs parameters/ search: index: test-index1 body: retriever: pinned: - ids: ["doc1"] * 101 + ids: ["doc1", "doc3"] + docs: [] retriever: standard: query: - match_all: {} \ No newline at end of file + match_all: {} + From 08e5addbe5c0d8ac3009231b46ced07d54588457 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 10 Apr 2025 22:24:12 +0000 Subject: [PATCH 17/83] [CI] Auto commit changes from spotless --- .../xpack/searchbusinessrules/SearchBusinessRules.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java index 30070321062cc..fcbb937e4b2a0 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -9,8 +9,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; -import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder; import java.util.List; From daa528e054d119687d58ff46dfe549c8a3e5f1ae Mon Sep 17 00:00:00 2001 From: Mridula Date: Fri, 11 Apr 2025 11:39:37 +0100 Subject: [PATCH 18/83] Resolved no source yaml error --- .../plugin/search-business-rules/build.gradle | 22 ------------------- ...rchBusinessRulesClientYamlTestSuiteIT.java | 18 +++++++++++++++ .../10_pinned_retriever.yml | 0 3 files changed, 18 insertions(+), 22 deletions(-) create mode 100644 x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java rename x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/{searchbusinessrules => search-business-rules}/10_pinned_retriever.yml (100%) diff --git a/x-pack/plugin/search-business-rules/build.gradle b/x-pack/plugin/search-business-rules/build.gradle index 0e021d4e7fe94..c0cfe92263a33 100644 --- a/x-pack/plugin/search-business-rules/build.gradle +++ b/x-pack/plugin/search-business-rules/build.gradle @@ -18,25 +18,9 @@ dependencies { testImplementation(testArtifact(project(':server'))) clusterModules project(':modules:mapper-extras') - clusterModules project(xpackModule('search-business-rules')) clusterModules project(':modules:lang-painless') } -restResources { - restApi { - include '_common', - 'bulk', - 'cluster', - 'indices', - 'index', - 'search', - 'xpack' - } - restTests { - includeXpack 'searchbusinessrules' - } -} - tasks.named("yamlRestTest") { usesDefaultDistribution("uses search business rules plugin") } @@ -44,9 +28,3 @@ tasks.named("yamlRestTest") { artifacts { restXpackTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test")) } - -testClusters.configureEach { - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.enabled', 'false' -} diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..1a16f9d029f02 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java @@ -0,0 +1,18 @@ +package org.elasticsearch.xpack.searchbusinessrules; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +public class SearchBusinessRulesClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public SearchBusinessRulesClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } +} \ No newline at end of file diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml similarity index 100% rename from x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/searchbusinessrules/10_pinned_retriever.yml rename to x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml From db1a1492bb5dc0add36c25855efa3e4fa54e53fe Mon Sep 17 00:00:00 2001 From: Mridula Date: Fri, 11 Apr 2025 11:59:28 +0100 Subject: [PATCH 19/83] Cluster loaded successfully --- .../searchbusinessrules/SearchBusinessRules.java | 10 ++++++++++ .../retriever/PinnedRetrieverBuilder.java | 9 +++++++-- .../org.elasticsearch.plugins.SearchPlugin | 1 + .../SearchBusinessRulesClientYamlTestSuiteIT.java | 15 +++++++++++++++ .../search-business-rules/10_pinned_retriever.yml | 4 ++-- 5 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.plugins.SearchPlugin diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java index fcbb937e4b2a0..4287149ced3e7 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -9,6 +9,9 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder; +import org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder; import java.util.List; @@ -21,4 +24,11 @@ public List> getQueries() { return singletonList(new QuerySpec<>(PinnedQueryBuilder.NAME, PinnedQueryBuilder::new, PinnedQueryBuilder::fromXContent)); } + @Override + public List> getRetrievers() { + return singletonList(new RetrieverSpec<>( + new ParseField(PinnedRetrieverBuilder.NAME), + PinnedRetrieverBuilder::fromXContent + )); + } } diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index a8ba331787f75..01a676767e82b 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -89,10 +89,15 @@ private void validateIdsAndDocs(List ids, List docs) } private void validateSort(SearchSourceBuilder source) { + // Allow null or empty sort, or sort only containing _score List> sorts = source.sorts(); - if (sorts != null && sorts.stream().anyMatch(sort -> sort instanceof ScoreSortBuilder == false)) { - throw new IllegalArgumentException("Pinned retriever only supports sorting by score. Custom sorting is not allowed."); + if (sorts != null && sorts.isEmpty() == false) { + // Check if there's any sort other than ScoreSortBuilder + if (sorts.stream().anyMatch(sort -> sort instanceof ScoreSortBuilder == false)) { + throw new IllegalArgumentException("Pinned retriever only supports sorting by score. Custom sorting is not allowed."); + } } + // If sorts is null, empty, or only contains ScoreSortBuilder, it's valid. } public PinnedRetrieverBuilder(List ids, List docs, RetrieverBuilder retrieverBuilder, int rankWindowSize) { diff --git a/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.plugins.SearchPlugin b/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.plugins.SearchPlugin new file mode 100644 index 0000000000000..d622876859864 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.plugins.SearchPlugin @@ -0,0 +1 @@ +org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRules diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java index 1a16f9d029f02..cd2e6628d6d16 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java @@ -2,11 +2,21 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class SearchBusinessRulesClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.security.enabled", "false") + .build(); + public SearchBusinessRulesClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -15,4 +25,9 @@ public SearchBusinessRulesClientYamlTestSuiteIT(final ClientYamlTestCandidate te public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } \ No newline at end of file diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index 8e6b106c1c21e..50cf5ddd487df 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -400,7 +400,7 @@ setup: --- "pinned retriever error cases": - do: - catch: /\[pinned\] duplicate id found in list: doc1/ + catch: '/\[pinned\] duplicate id found in list: doc1/' search: index: test-index1 body: @@ -413,7 +413,7 @@ setup: match_all: {} - do: - catch: /\[pinned\] Cannot specify both ids and docs parameters/ + catch: '/\[pinned\] Cannot specify both ids and docs parameters/' search: index: test-index1 body: From e00c2496de28d5a22d43310ecf49a6361fa4538d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 14 Apr 2025 01:27:47 +0000 Subject: [PATCH 20/83] [CI] Auto commit changes from spotless --- .../xpack/searchbusinessrules/SearchBusinessRules.java | 6 +----- .../SearchBusinessRulesClientYamlTestSuiteIT.java | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java index 4287149ced3e7..609e50ec49f91 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -10,7 +10,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder; import org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder; import java.util.List; @@ -26,9 +25,6 @@ public List> getQueries() { @Override public List> getRetrievers() { - return singletonList(new RetrieverSpec<>( - new ParseField(PinnedRetrieverBuilder.NAME), - PinnedRetrieverBuilder::fromXContent - )); + return singletonList(new RetrieverSpec<>(new ParseField(PinnedRetrieverBuilder.NAME), PinnedRetrieverBuilder::fromXContent)); } } diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java index cd2e6628d6d16..e12a55c6fe767 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java @@ -30,4 +30,4 @@ public static Iterable parameters() throws Exception { protected String getTestRestCluster() { return cluster.getHttpAddresses(); } -} \ No newline at end of file +} From c659c1c365101e17b20b5b551e1cf738e5ec9ed3 Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 14 Apr 2025 09:00:18 +0100 Subject: [PATCH 21/83] Fixed integration test --- .../test/search-business-rules/10_pinned_retriever.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index 50cf5ddd487df..aa5cc48d4dd35 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -400,7 +400,7 @@ setup: --- "pinned retriever error cases": - do: - catch: '/\[pinned\] duplicate id found in list: doc1/' + catch: '/\[Pinned\] duplicate id found in list: doc1/' search: index: test-index1 body: From 7ab4b76faca635c785808ebc0c923e6d39b4c7b5 Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 14 Apr 2025 10:34:13 +0100 Subject: [PATCH 22/83] Made validate sort less strict --- .../retriever/PinnedRetrieverBuilder.java | 10 +- .../10_pinned_retriever.yml | 180 ++++-------------- 2 files changed, 44 insertions(+), 146 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index 01a676767e82b..dae1d48f1c2d9 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -89,15 +89,13 @@ private void validateIdsAndDocs(List ids, List docs) } private void validateSort(SearchSourceBuilder source) { - // Allow null or empty sort, or sort only containing _score List> sorts = source.sorts(); if (sorts != null && sorts.isEmpty() == false) { - // Check if there's any sort other than ScoreSortBuilder - if (sorts.stream().anyMatch(sort -> sort instanceof ScoreSortBuilder == false)) { - throw new IllegalArgumentException("Pinned retriever only supports sorting by score. Custom sorting is not allowed."); - } + return; + } + if (sorts.stream().anyMatch(sort -> sort instanceof ScoreSortBuilder == false)) { + throw new IllegalArgumentException("Pinned retriever only supports sorting by score. Custom sorting is not allowed."); } - // If sorts is null, empty, or only contains ScoreSortBuilder, it's valid. } public PinnedRetrieverBuilder(List ids, List docs, RetrieverBuilder retrieverBuilder, int rankWindowSize) { diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index aa5cc48d4dd35..c2f5e153fea05 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -4,9 +4,8 @@ setup: index: test-index1 body: settings: - index: - number_of_shards: 1 - number_of_replicas: 0 + number_of_shards: 1 + number_of_replicas: 0 - do: bulk: @@ -49,11 +48,6 @@ setup: - match: { hits.hits.2._id: doc3 } - match: { hits.hits.3._id: doc4 } - match: { hits.hits.4._id: doc5 } - - is_true: hits.hits.0._score > 1.0 - - is_true: hits.hits.1._score > 1.0 - - is_true: hits.hits.2._score <= 1.0 - - is_true: hits.hits.3._score <= 1.0 - - is_true: hits.hits.4._score <= 1.0 --- "pinned retriever parameter variations": @@ -64,10 +58,10 @@ setup: retriever: pinned: docs: - - index: test-index1 - id: doc1 - - index: test-index1 - id: doc2 + - _index: test-index1 + _id: doc1 + - _index: test-index1 + _id: doc2 retriever: standard: query: @@ -76,11 +70,7 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } - - is_true: hits.hits.0._score > 1.0 - - is_true: hits.hits.1._score > 1.0 - - is_true: hits.hits.2._score <= 1.0 - - is_true: hits.hits.3._score <= 1.0 - - is_true: hits.hits.4._score <= 1.0 + - match: { hits.hits.2._id: doc3 } - do: search: @@ -88,7 +78,7 @@ setup: body: retriever: pinned: - id: "doc1" + ids: ["doc1"] retriever: standard: query: @@ -96,11 +86,7 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - - is_true: hits.hits.0._score > 1.0 - - is_true: hits.hits.1._score <= 1.0 - - is_true: hits.hits.2._score <= 1.0 - - is_true: hits.hits.3._score <= 1.0 - - is_true: hits.hits.4._score <= 1.0 + - match: { hits.hits.1._id: doc2 } - do: search: @@ -108,9 +94,9 @@ setup: body: retriever: pinned: - doc: - index: test-index1 - id: doc1 + docs: + - _index: test-index1 + _id: doc1 retriever: standard: query: @@ -118,11 +104,7 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - - is_true: hits.hits.0._score > 1.0 - - is_true: hits.hits.1._score <= 1.0 - - is_true: hits.hits.2._score <= 1.0 - - is_true: hits.hits.3._score <= 1.0 - - is_true: hits.hits.4._score <= 1.0 + - match: { hits.hits.1._id: doc2 } --- "pinned retriever with combined parameters": @@ -132,10 +114,7 @@ setup: body: retriever: pinned: - id: "doc1" - doc: - index: test-index1 - id: doc2 + ids: ["doc1", "doc2"] retriever: standard: query: @@ -144,11 +123,7 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } - - is_true: hits.hits.0._score > 1.0 - - is_true: hits.hits.1._score > 1.0 - - is_true: hits.hits.2._score <= 1.0 - - is_true: hits.hits.3._score <= 1.0 - - is_true: hits.hits.4._score <= 1.0 + - match: { hits.hits.2._id: doc3 } - do: search: @@ -157,11 +132,6 @@ setup: retriever: pinned: ids: ["doc1", "doc3"] - docs: - - index: test-index1 - id: doc2 - - index: test-index1 - id: doc4 retriever: standard: query: @@ -172,11 +142,7 @@ setup: - match: { hits.hits.1._id: doc3 } - match: { hits.hits.2._id: doc2 } - match: { hits.hits.3._id: doc4 } - - is_true: hits.hits.0._score > 1.0 - - is_true: hits.hits.1._score > 1.0 - - is_true: hits.hits.2._score > 1.0 - - is_true: hits.hits.3._score > 1.0 - - is_true: hits.hits.4._score <= 1.0 + - match: { hits.hits.4._id: doc5 } --- "pinned retriever combined with rrf": @@ -189,34 +155,23 @@ setup: ids: ["doc1", "doc2"] retriever: rrf: - retrievers: [ - { - standard: { - query: { + retrievers: + - + standard: + query: match: { text: "document one" } - } - } - }, - { - standard: { - query: { + - + standard: + query: match: { text: "document two" } - } - } - } - ] - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } - - is_true: hits.hits.0._score > 1.0 - - is_true: hits.hits.1._score > 1.0 - - is_true: hits.hits.2._score > hits.hits.3._score - - is_true: hits.hits.3._score > hits.hits.4._score --- "pinned retriever with pagination": @@ -237,7 +192,6 @@ setup: - match: { hits.total.value: 5 } - length: { hits.hits: 1 } - match: { hits.hits.0._id: doc2 } - - is_true: hits.hits.0._score > 1.0 --- "pinned retriever as a sub-retriever": @@ -247,34 +201,23 @@ setup: body: retriever: rrf: - retrievers: [ - { - standard: { - query: { + retrievers: + - + standard: + query: match_all: {} - } - } - }, - { - pinned: { - ids: ["doc1", "doc2"], - retriever: { - standard: { - query: { + - + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: match_all: {} - } - } - } - } - } - ] - match: { hits.total.value: 5 } - - is_true: hits.hits.0._score > 1.0 - - is_true: hits.hits.1._score > 1.0 - - is_true: hits.hits.2._score <= 1.0 - - is_true: hits.hits.3._score <= 1.0 - - is_true: hits.hits.4._score <= 1.0 + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.2._id: doc3 } --- "pinned retriever with explicit sort on score": @@ -294,11 +237,7 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } - - is_true: hits.hits.0._score > 1.0 - - is_true: hits.hits.1._score > 1.0 - - is_true: hits.hits.2._score <= 1.0 - - is_true: hits.hits.3._score <= 1.0 - - is_true: hits.hits.4._score <= 1.0 + - match: { hits.hits.2._id: doc3 } --- "pinned retriever with rank window size": @@ -323,7 +262,6 @@ setup: - match: { hits.total.value: 5 } - length: { hits.hits: 1 } - match: { hits.hits.0._id: doc1 } - - is_true: hits.hits.0._score > 1.0 - do: headers: @@ -344,11 +282,7 @@ setup: - length: { hits.hits: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } - - is_true: hits.hits.0._score > 1.0 - - is_true: hits.hits.1._score > 1.0 - - is_true: hits.hits.2._score <= 1.0 - - is_true: hits.hits.3._score <= 1.0 - - is_true: hits.hits.4._score <= 1.0 + - match: { hits.hits.2._id: doc3 } --- "pinned retriever explanation": @@ -366,15 +300,11 @@ setup: explain: true - match: { hits.hits.0._id: doc1 } - - is_true: hits.hits.0._explanation.value > 1.0 - - match: { hits.hits.0._explanation.description: "pinned documents" } - - match: { hits.hits.0._explanation.details.0.details.0.details.0.description: "pinned_by: [ids]" } + - match: { hits.hits.0._explanation.description: "/doc \\[\\d+\\] with an original score of \\[-?[\\d.E]+\\] is at rank \\[\\d+\\] from the following source queries\\./" } - match: { hits.hits.0._explanation.details.0.details.0.details.1.description: "is_pinned: true" } - match: { hits.hits.1._id: doc2 } - - is_true: hits.hits.1._explanation.value > 1.0 - - match: { hits.hits.1._explanation.description: "pinned documents" } - - match: { hits.hits.1._explanation.details.0.details.0.details.0.description: "pinned_by: [ids]" } + - match: { hits.hits.1._explanation.description: "/doc \\[\\d+\\] with an original score of \\[-?[\\d.E]+\\] is at rank \\[\\d+\\] from the following source queries\\./" } - match: { hits.hits.1._explanation.details.0.details.0.details.1.description: "is_pinned: true" } --- @@ -396,33 +326,3 @@ setup: - match: { hits.hits.2._id: doc3 } - match: { hits.hits.3._id: doc4 } - match: { hits.hits.4._id: doc5 } - ---- -"pinned retriever error cases": - - do: - catch: '/\[Pinned\] duplicate id found in list: doc1/' - search: - index: test-index1 - body: - retriever: - pinned: - ids: ["doc1", "doc1"] - retriever: - standard: - query: - match_all: {} - - - do: - catch: '/\[pinned\] Cannot specify both ids and docs parameters/' - search: - index: test-index1 - body: - retriever: - pinned: - ids: ["doc1", "doc3"] - docs: [] - retriever: - standard: - query: - match_all: {} - From e1a4362cfa7382344f2c80d53ca4fefbfd739e48 Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 14 Apr 2025 12:48:19 +0100 Subject: [PATCH 23/83] Included shareddoc sorting in the validate sort --- .../retriever/PinnedRetrieverBuilder.java | 19 +++- .../10_pinned_retriever.yml | 96 +++++++++++++++++++ 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index dae1d48f1c2d9..de01e643d1f2c 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -17,7 +17,9 @@ import org.elasticsearch.search.retriever.RetrieverBuilder; import org.elasticsearch.search.retriever.RetrieverBuilderWrapper; import org.elasticsearch.search.retriever.RetrieverParserContext; +import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.ShardDocSortField; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -90,11 +92,22 @@ private void validateIdsAndDocs(List ids, List docs) private void validateSort(SearchSourceBuilder source) { List> sorts = source.sorts(); - if (sorts != null && sorts.isEmpty() == false) { + if (sorts == null || sorts.isEmpty()) { return; } - if (sorts.stream().anyMatch(sort -> sort instanceof ScoreSortBuilder == false)) { - throw new IllegalArgumentException("Pinned retriever only supports sorting by score. Custom sorting is not allowed."); + for (SortBuilder sort : sorts) { + if (sort instanceof ScoreSortBuilder) { + continue; + } + if (sort instanceof FieldSortBuilder) { + FieldSortBuilder fieldSort = (FieldSortBuilder) sort; + if (ShardDocSortField.NAME.equals(fieldSort.getFieldName())) { + continue; + } + } + throw new IllegalArgumentException( + "Pinned retriever only supports sorting by score. Custom sorting is not allowed." + ); } } diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index c2f5e153fea05..10b4c0e599639 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -172,6 +172,9 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } + - is_true: hits.hits.0._score + - is_true: hits.hits.1._score + - is_true: hits.hits.0._score >= hits.hits.1._score --- "pinned retriever with pagination": @@ -326,3 +329,96 @@ setup: - match: { hits.hits.2._id: doc3 } - match: { hits.hits.3._id: doc4 } - match: { hits.hits.4._id: doc5 } + +--- +"pinned retriever error case - both ids and docs": + - do: + # Expect a bad_request error with this message + catch: /Both 'ids' and 'docs' cannot be specified at the same time/ + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1"] # Provide some IDs + docs: # Provide some docs + - _index: test-index1 + _id: doc2 + retriever: + standard: + query: + match_all: {} + +--- +"pinned retriever error case - invalid sort": + - do: + # Expect the earlier validation error + catch: /cannot specify \[retriever\] and \[sort\]/ + search: + index: test-index1 + body: + sort: [ "text" ] + retriever: + pinned: + ids: ["doc1"] + retriever: + standard: + query: + match_all: {} + +--- +"pinned retriever error case - duplicate id": + - do: + # Catch duplicate ID error + catch: /duplicate id found/ + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc1"] + retriever: + standard: + query: + match_all: {} + +--- +"pinned retriever with empty docs array": + - do: + search: + index: test-index1 + body: + retriever: + pinned: + docs: [] # Empty docs array + retriever: + standard: + query: + match_all: {} + # Expect regular match_all results, no pinning effect + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.2._id: doc3 } + - match: { hits.hits.3._id: doc4 } + - match: { hits.hits.4._id: doc5 } + +--- +"pinned retriever with ids and empty docs array": + - do: + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1"] # Provide IDs + docs: [] # Provide empty docs array + retriever: + standard: + query: + match_all: {} + # Expect pinning based on "ids", empty "docs" is ignored + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } # doc1 should be pinned + - match: { hits.hits.1._id: doc2 } # Rest follow + - match: { hits.hits.2._id: doc3 } From 3e40f6ef27fe24fe1a4df873e204e871fde99ad7 Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 14 Apr 2025 16:04:28 +0100 Subject: [PATCH 24/83] All yaml issues resolved --- = | 0 .../10_pinned_retriever.yml | 169 ++++++++---------- 2 files changed, 73 insertions(+), 96 deletions(-) create mode 100644 = diff --git a/= b/= new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index 10b4c0e599639..3d9681c86bc1e 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -28,27 +28,6 @@ setup: _id: doc5 - { "text": "document five" } ---- -"pinned retriever basic functionality": - - do: - search: - index: test-index1 - body: - retriever: - pinned: - ids: ["doc1", "doc2"] - retriever: - standard: - query: - match_all: {} - - - match: { hits.total.value: 5 } - - match: { hits.hits.0._id: doc1 } - - match: { hits.hits.1._id: doc2 } - - match: { hits.hits.2._id: doc3 } - - match: { hits.hits.3._id: doc4 } - - match: { hits.hits.4._id: doc5 } - --- "pinned retriever parameter variations": - do: @@ -107,8 +86,12 @@ setup: - match: { hits.hits.1._id: doc2 } --- -"pinned retriever with combined parameters": +"pinned retriever dynamic pinning and ordering": + - skip: + features: headers - do: + headers: + Content-Type: application/json search: index: test-index1 body: @@ -119,13 +102,19 @@ setup: standard: query: match_all: {} + explain: true - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } - match: { hits.hits.2._id: doc3 } + - match: { hits.hits.0._score: 1.7014124E38 } + - match: { hits.hits.1._score: 1.7014122E38 } + - match: { hits.hits.2._score: 1.0 } - do: + headers: + Content-Type: application/json search: index: test-index1 body: @@ -136,6 +125,7 @@ setup: standard: query: match_all: {} + explain: true - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } @@ -143,38 +133,50 @@ setup: - match: { hits.hits.2._id: doc2 } - match: { hits.hits.3._id: doc4 } - match: { hits.hits.4._id: doc5 } + - match: { hits.hits.0._score: 1.7014124E38 } + - match: { hits.hits.1._score: 1.7014122E38 } + - match: { hits.hits.2._score: 1.0 } --- "pinned retriever combined with rrf": + - skip: + features: headers + - do: + headers: + Content-Type: application/json search: index: test-index1 body: retriever: pinned: - ids: ["doc1", "doc2"] + ids: ["doc1"] retriever: rrf: - retrievers: - - - standard: - query: - match: { - text: "document one" - } - - - standard: - query: - match: { - text: "document two" - } + retrievers: [ + { + standard: { + query: { + term: { text: "document" } + } + } + }, + { + standard: { + query: { + term: { text: "three" } + } + } + } + ] + rank_window_size: 10 - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - - match: { hits.hits.1._id: doc2 } - - is_true: hits.hits.0._score - - is_true: hits.hits.1._score - - is_true: hits.hits.0._score >= hits.hits.1._score + - match: { hits.hits.0._score: 1.7014122E38 } + - match: { hits.hits.1._id: doc3 } + - match: { hits.hits.2._id: doc2 } + - match: { hits.hits.1._score < hits.hits.0._score } --- "pinned retriever with pagination": @@ -287,28 +289,28 @@ setup: - match: { hits.hits.1._id: doc2 } - match: { hits.hits.2._id: doc3 } ---- -"pinned retriever explanation": - - do: - search: - index: test-index1 - body: - retriever: - pinned: - ids: ["doc1", "doc2"] - retriever: - standard: - query: - match_all: {} - explain: true - - - match: { hits.hits.0._id: doc1 } - - match: { hits.hits.0._explanation.description: "/doc \\[\\d+\\] with an original score of \\[-?[\\d.E]+\\] is at rank \\[\\d+\\] from the following source queries\\./" } - - match: { hits.hits.0._explanation.details.0.details.0.details.1.description: "is_pinned: true" } - - - match: { hits.hits.1._id: doc2 } - - match: { hits.hits.1._explanation.description: "/doc \\[\\d+\\] with an original score of \\[-?[\\d.E]+\\] is at rank \\[\\d+\\] from the following source queries\\./" } - - match: { hits.hits.1._explanation.details.0.details.0.details.1.description: "is_pinned: true" } +# --- +# "pinned retriever explanation": +# - do: +# search: +# index: test-index1 +# body: +# retriever: +# pinned: +# ids: ["doc1", "doc2"] +# retriever: +# standard: +# query: +# match_all: {} +# explain: true + +# - match: { hits.hits.0._id: doc1 } +# - match: { hits.hits.0._explanation.description: /^doc.*queries\.$/ } +# - match: { hits.hits.0._explanation.details.0.details.0.details.1.description: "is_pinned: true" } + +# - match: { hits.hits.1._id: doc2 } +# - match: { hits.hits.1._explanation.description: /^doc.*queries\.$/ } +# - match: { hits.hits.1._explanation.details.0.details.0.details.1.description: "is_pinned: true" } --- "pinned retriever with empty parameters": @@ -333,15 +335,14 @@ setup: --- "pinned retriever error case - both ids and docs": - do: - # Expect a bad_request error with this message catch: /Both 'ids' and 'docs' cannot be specified at the same time/ search: index: test-index1 body: retriever: pinned: - ids: ["doc1"] # Provide some IDs - docs: # Provide some docs + ids: ["doc1"] + docs: - _index: test-index1 _id: doc2 retriever: @@ -349,27 +350,9 @@ setup: query: match_all: {} ---- -"pinned retriever error case - invalid sort": - - do: - # Expect the earlier validation error - catch: /cannot specify \[retriever\] and \[sort\]/ - search: - index: test-index1 - body: - sort: [ "text" ] - retriever: - pinned: - ids: ["doc1"] - retriever: - standard: - query: - match_all: {} - --- "pinned retriever error case - duplicate id": - do: - # Catch duplicate ID error catch: /duplicate id found/ search: index: test-index1 @@ -383,42 +366,36 @@ setup: match_all: {} --- -"pinned retriever with empty docs array": +"pinned retriever with ids and empty docs array": - do: search: index: test-index1 body: retriever: pinned: - docs: [] # Empty docs array + ids: ["doc1"] + docs: [] retriever: standard: query: match_all: {} - # Expect regular match_all results, no pinning effect - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } - match: { hits.hits.2._id: doc3 } - - match: { hits.hits.3._id: doc4 } - - match: { hits.hits.4._id: doc5 } --- -"pinned retriever with ids and empty docs array": +"pinned retriever errors if sorting by anything other than score": - do: + catch: /Pinned retriever only supports sorting by score. Custom sorting is not allowed./ search: index: test-index1 body: retriever: pinned: - ids: ["doc1"] # Provide IDs - docs: [] # Provide empty docs array + ids: ["doc1"] retriever: standard: query: match_all: {} - # Expect pinning based on "ids", empty "docs" is ignored - - match: { hits.total.value: 5 } - - match: { hits.hits.0._id: doc1 } # doc1 should be pinned - - match: { hits.hits.1._id: doc2 } # Rest follow - - match: { hits.hits.2._id: doc3 } + sort: [ { "_id": "desc" } ] From 1d4eb6f8c8910d68d30619758782c336ea5d5fea Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 14 Apr 2025 17:02:00 +0100 Subject: [PATCH 25/83] Explanation yaml files added --- .../retriever/PinnedRetrieverBuilder.java | 12 ++--- .../10_pinned_retriever.yml | 49 ++++++++++--------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index de01e643d1f2c..50689a3682c38 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -100,14 +100,12 @@ private void validateSort(SearchSourceBuilder source) { continue; } if (sort instanceof FieldSortBuilder) { - FieldSortBuilder fieldSort = (FieldSortBuilder) sort; - if (ShardDocSortField.NAME.equals(fieldSort.getFieldName())) { - continue; - } + FieldSortBuilder fieldSort = (FieldSortBuilder) sort; + if (ShardDocSortField.NAME.equals(fieldSort.getFieldName())) { + continue; + } } - throw new IllegalArgumentException( - "Pinned retriever only supports sorting by score. Custom sorting is not allowed." - ); + throw new IllegalArgumentException("Pinned retriever only supports sorting by score. Custom sorting is not allowed."); } } diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index 3d9681c86bc1e..79cf6739ada4f 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -289,28 +289,33 @@ setup: - match: { hits.hits.1._id: doc2 } - match: { hits.hits.2._id: doc3 } -# --- -# "pinned retriever explanation": -# - do: -# search: -# index: test-index1 -# body: -# retriever: -# pinned: -# ids: ["doc1", "doc2"] -# retriever: -# standard: -# query: -# match_all: {} -# explain: true - -# - match: { hits.hits.0._id: doc1 } -# - match: { hits.hits.0._explanation.description: /^doc.*queries\.$/ } -# - match: { hits.hits.0._explanation.details.0.details.0.details.1.description: "is_pinned: true" } - -# - match: { hits.hits.1._id: doc2 } -# - match: { hits.hits.1._explanation.description: /^doc.*queries\.$/ } -# - match: { hits.hits.1._explanation.details.0.details.0.details.1.description: "is_pinned: true" } +--- +"pinned retriever explanation": + - skip: + features: headers + + - do: + headers: + Content-Type: application/json + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: + match_all: {} + explain: true + + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._explanation.value: 1.0 } + - match: { hits.hits.0._explanation.description: "doc [0] with an original score of [1.7014124E38] is at rank [1] from the following source queries." } + + - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.1._explanation.value: 2.0 } + - match: { hits.hits.1._explanation.description: "doc [1] with an original score of [1.7014122E38] is at rank [2] from the following source queries." } --- "pinned retriever with empty parameters": From 05a05a0cb5771f5e2938938926eca235df6d37d3 Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 14 Apr 2025 18:47:01 +0100 Subject: [PATCH 26/83] Trying to add clustering to the test --- .../plugin/search-business-rules/build.gradle | 3 -- .../src/main/java/module-info.java | 2 ++ .../SearchBusinessRules.java | 2 ++ .../SearchBusinessRulesFeatures.java | 30 +++++++++++++++++++ ...lasticsearch.features.FeatureSpecification | 1 + 5 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java create mode 100644 x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification diff --git a/x-pack/plugin/search-business-rules/build.gradle b/x-pack/plugin/search-business-rules/build.gradle index c0cfe92263a33..ca9cd6f60517a 100644 --- a/x-pack/plugin/search-business-rules/build.gradle +++ b/x-pack/plugin/search-business-rules/build.gradle @@ -16,9 +16,6 @@ dependencies { compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) testImplementation(testArtifact(project(':server'))) - - clusterModules project(':modules:mapper-extras') - clusterModules project(':modules:lang-painless') } tasks.named("yamlRestTest") { diff --git a/x-pack/plugin/search-business-rules/src/main/java/module-info.java b/x-pack/plugin/search-business-rules/src/main/java/module-info.java index d11c809b43854..e25d55aec1dc9 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/module-info.java +++ b/x-pack/plugin/search-business-rules/src/main/java/module-info.java @@ -16,4 +16,6 @@ requires org.elasticsearch.xcore; exports org.elasticsearch.xpack.searchbusinessrules; + provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRulesFeatures; + } diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java index 609e50ec49f91..ade18436c5a91 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.searchbusinessrules; +import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.xcontent.ParseField; @@ -27,4 +28,5 @@ public List> getQueries() { public List> getRetrievers() { return singletonList(new RetrieverSpec<>(new ParseField(PinnedRetrieverBuilder.NAME), PinnedRetrieverBuilder::fromXContent)); } + } diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java new file mode 100644 index 0000000000000..e8e66e676ddef --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + + package org.elasticsearch.xpack.searchbusinessrules; + + import org.elasticsearch.features.FeatureSpecification; + import org.elasticsearch.features.NodeFeature; + + import java.util.Set; + + public class SearchBusinessRulesFeatures implements FeatureSpecification { + + public static final NodeFeature PINNED_RETRIEVER_FEATURE = new NodeFeature( + "pinned_retriever"); + + @Override + public Set getFeatures() { + return Set.of(); + } + + @Override + public Set getTestFeatures() { + return Set.of(PINNED_RETRIEVER_FEATURE); + } + } + \ No newline at end of file diff --git a/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification new file mode 100644 index 0000000000000..4d8cab289f9a4 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -0,0 +1 @@ +org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRulesFeatures \ No newline at end of file From e2fc29ba6211d07421bacc66b082bf967064c7d4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 14 Apr 2025 17:55:12 +0000 Subject: [PATCH 27/83] [CI] Auto commit changes from spotless --- .../src/main/java/module-info.java | 1 + .../SearchBusinessRules.java | 1 - .../SearchBusinessRulesFeatures.java | 42 +++++++++---------- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/module-info.java b/x-pack/plugin/search-business-rules/src/main/java/module-info.java index e25d55aec1dc9..fa736fe8c399b 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/module-info.java +++ b/x-pack/plugin/search-business-rules/src/main/java/module-info.java @@ -16,6 +16,7 @@ requires org.elasticsearch.xcore; exports org.elasticsearch.xpack.searchbusinessrules; + provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRulesFeatures; } diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java index ade18436c5a91..522a0e76ff89c 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.searchbusinessrules; -import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.xcontent.ParseField; diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java index e8e66e676ddef..44db2323ef65e 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java @@ -5,26 +5,24 @@ * 2.0. */ - package org.elasticsearch.xpack.searchbusinessrules; +package org.elasticsearch.xpack.searchbusinessrules; - import org.elasticsearch.features.FeatureSpecification; - import org.elasticsearch.features.NodeFeature; - - import java.util.Set; - - public class SearchBusinessRulesFeatures implements FeatureSpecification { - - public static final NodeFeature PINNED_RETRIEVER_FEATURE = new NodeFeature( - "pinned_retriever"); - - @Override - public Set getFeatures() { - return Set.of(); - } - - @Override - public Set getTestFeatures() { - return Set.of(PINNED_RETRIEVER_FEATURE); - } - } - \ No newline at end of file +import org.elasticsearch.features.FeatureSpecification; +import org.elasticsearch.features.NodeFeature; + +import java.util.Set; + +public class SearchBusinessRulesFeatures implements FeatureSpecification { + + public static final NodeFeature PINNED_RETRIEVER_FEATURE = new NodeFeature("pinned_retriever"); + + @Override + public Set getFeatures() { + return Set.of(); + } + + @Override + public Set getTestFeatures() { + return Set.of(PINNED_RETRIEVER_FEATURE); + } +} From dc02f53a6a269cf618ef185d6273609326dab66c Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 14 Apr 2025 19:43:49 +0100 Subject: [PATCH 28/83] Removed explicit feature specification --- .../services/org.elasticsearch.features.FeatureSpecification | 1 - 1 file changed, 1 deletion(-) delete mode 100644 x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification diff --git a/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification deleted file mode 100644 index 4d8cab289f9a4..0000000000000 --- a/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification +++ /dev/null @@ -1 +0,0 @@ -org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRulesFeatures \ No newline at end of file From e2b8cb6e53d4bf8cfd426456f56df8691e9e6f0d Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 14 Apr 2025 19:47:18 +0100 Subject: [PATCH 29/83] Deleted the empty file --- = | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 = diff --git a/= b/= deleted file mode 100644 index e69de29bb2d1d..0000000000000 From f621af02923bd029dfe75d737d68638c6395322a Mon Sep 17 00:00:00 2001 From: Mridula Date: Tue, 15 Apr 2025 22:27:36 +0100 Subject: [PATCH 30/83] Put the node feature in the proper place --- query.es | 65 +++++ result.json | 223 ++++++++++++++++++ .../SearchBusinessRulesFeatures.java | 4 +- .../retriever/PinnedRetrieverBuilder.java | 3 + 4 files changed, 293 insertions(+), 2 deletions(-) create mode 100644 query.es create mode 100644 result.json diff --git a/query.es b/query.es new file mode 100644 index 0000000000000..b2c5f970a678a --- /dev/null +++ b/query.es @@ -0,0 +1,65 @@ +% Title: Elasticsearch Query with Pinned Documents + + +POST my-index/_doc/1 +{ "title": "Beginner's Guide to ML" } + +POST my-index/_doc/2 +{ "title": "Sponsored ML Course" } + +POST my-index/_doc/3 +{ "title": "Advanced Topics in Machine Learning" } + + +GET my-index/_search +{ + "query": { + "match": { + "title": "machine learning" + } + } +} + +GET my-index/_search +{ + "query": { + "pinned": { + "ids": ["1", "2"], + "organic": { + "match": { + "title": "machine learning" + } + } + } + } +} + +GET my-index/_search +{ + "query": { + "pinned": { + "ids": ["doc-id-1", "doc-id-2"], // pinned to top + "organic": { + "match": { + "title": "machine learning" + } + } + } + } +} + +GET my-index/_search +{ + "explain": true, + "query": { + "pinned": { + "ids": ["1", "2"], + "organic": { + "match": { + "title": "machine learning" + } + } + } + } +} + diff --git a/result.json b/result.json new file mode 100644 index 0000000000000..85e54ebd82d6d --- /dev/null +++ b/result.json @@ -0,0 +1,223 @@ +{ + "took": 30, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 3, + "relation": "eq" + }, + "max_score": 1.7014124e+38, + "hits": [ + { + "_shard": "[my-index][0]", + "_node": "UbGQ3NQ0RMaG4J2rApFQPw", + "_index": "my-index", + "_id": "1", + "_score": 1.7014124e+38, + "_source": { + "title": "Beginner's Guide to ML" + }, + "_explanation": { + "value": 1.7014124e+38, + "description": "max of:", + "details": [ + { + "value": 1.7014124e+38, + "description": "max of:", + "details": [ + { + "value": 1.7014124e+38, + "description": "ConstantScore(_id:([fe 1f]))^1.7014124E38", + "details": [] + } + ] + } + ] + } + }, + { + "_shard": "[my-index][0]", + "_node": "UbGQ3NQ0RMaG4J2rApFQPw", + "_index": "my-index", + "_id": "2", + "_score": 1.7014122e+38, + "_source": { + "title": "Sponsored ML Course" + }, + "_explanation": { + "value": 1.7014122e+38, + "description": "max of:", + "details": [ + { + "value": 1.7014122e+38, + "description": "max of:", + "details": [ + { + "value": 1.7014122e+38, + "description": "ConstantScore(_id:([fe 2f]))^1.7014122E38", + "details": [] + } + ] + } + ] + } + }, + { + "_shard": "[my-index][0]", + "_node": "UbGQ3NQ0RMaG4J2rApFQPw", + "_index": "my-index", + "_id": "3", + "_score": 1.7796488, + "_source": { + "title": "Advanced Topics in Machine Learning" + }, + "_explanation": { + "value": 1.7796488, + "description": "max of:", + "details": [ + { + "value": 1.7796488, + "description": "sum of:", + "details": [ + { + "value": 0.8898244, + "description": "weight(title:machine in 2) [PerFieldSimilarity], result of:", + "details": [ + { + "value": 0.8898244, + "description": "score(freq=1.0), computed as boost * idf * tf from:", + "details": [ + { + "value": 2.2, + "description": "boost", + "details": [] + }, + { + "value": 0.98082924, + "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:", + "details": [ + { + "value": 1, + "description": "n, number of documents containing term", + "details": [] + }, + { + "value": 3, + "description": "N, total number of documents with field", + "details": [] + } + ] + }, + { + "value": 0.4123711, + "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:", + "details": [ + { + "value": 1, + "description": "freq, occurrences of term within document", + "details": [] + }, + { + "value": 1.2, + "description": "k1, term saturation parameter", + "details": [] + }, + { + "value": 0.75, + "description": "b, length normalization parameter", + "details": [] + }, + { + "value": 5, + "description": "dl, length of field", + "details": [] + }, + { + "value": 4, + "description": "avgdl, average length of field", + "details": [] + } + ] + } + ] + } + ] + }, + { + "value": 0.8898244, + "description": "weight(title:learning in 2) [PerFieldSimilarity], result of:", + "details": [ + { + "value": 0.8898244, + "description": "score(freq=1.0), computed as boost * idf * tf from:", + "details": [ + { + "value": 2.2, + "description": "boost", + "details": [] + }, + { + "value": 0.98082924, + "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:", + "details": [ + { + "value": 1, + "description": "n, number of documents containing term", + "details": [] + }, + { + "value": 3, + "description": "N, total number of documents with field", + "details": [] + } + ] + }, + { + "value": 0.4123711, + "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:", + "details": [ + { + "value": 1, + "description": "freq, occurrences of term within document", + "details": [] + }, + { + "value": 1.2, + "description": "k1, term saturation parameter", + "details": [] + }, + { + "value": 0.75, + "description": "b, length normalization parameter", + "details": [] + }, + { + "value": 5, + "description": "dl, length of field", + "details": [] + }, + { + "value": 4, + "description": "avgdl, average length of field", + "details": [] + } + ] + } + ] + } + ] + } + ] + } + ] + } + } + ] + } +} \ No newline at end of file diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java index 44db2323ef65e..d702a0fcc6ea0 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java @@ -12,9 +12,9 @@ import java.util.Set; -public class SearchBusinessRulesFeatures implements FeatureSpecification { +import static org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder.PINNED_RETRIEVER_FEATURE; - public static final NodeFeature PINNED_RETRIEVER_FEATURE = new NodeFeature("pinned_retriever"); +public class SearchBusinessRulesFeatures implements FeatureSpecification { @Override public Set getFeatures() { diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index 50689a3682c38..8f32be48738ac 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -9,6 +9,7 @@ import org.apache.lucene.search.ScoreDoc; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -49,6 +50,8 @@ public final class PinnedRetrieverBuilder extends CompoundRetrieverBuilder PARSER = new ConstructingObjectParser<>( NAME, From a3de6362425da7b6d51a0bf5ac5be66baafa09ed Mon Sep 17 00:00:00 2001 From: Mridula Date: Tue, 15 Apr 2025 22:35:40 +0100 Subject: [PATCH 31/83] Added additional validation in ids and docs --- .../retriever/PinnedRetrieverBuilder.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index 8f32be48738ac..81574a8452b76 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -88,9 +88,15 @@ public static PinnedRetrieverBuilder fromXContent(XContentParser parser, Retriev private final List docs; private void validateIdsAndDocs(List ids, List docs) { - if ((ids != null && ids.isEmpty() == false) && (docs != null && docs.isEmpty() == false)) { + boolean hasIds = ids != null && ids.isEmpty() == false; + boolean hasDocs = docs != null && docs.isEmpty() == false; + + if (hasIds && hasDocs) { throw new IllegalArgumentException("Both 'ids' and 'docs' cannot be specified at the same time"); } + if (hasIds == false && hasDocs == false) { + throw new IllegalArgumentException("Either 'ids' or 'docs' must be provided and non-empty for pinned retriever"); + } } private void validateSort(SearchSourceBuilder source) { From 5626dd7b75675cdaee3d934df3787c999f446cf5 Mon Sep 17 00:00:00 2001 From: Mridula Date: Tue, 15 Apr 2025 23:00:33 +0100 Subject: [PATCH 32/83] Cleaned the create pinned query validation --- .../retriever/PinnedRetrieverBuilder.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index 81574a8452b76..1aeba640a0223 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -114,7 +114,9 @@ private void validateSort(SearchSourceBuilder source) { continue; } } - throw new IllegalArgumentException("Pinned retriever only supports sorting by score. Custom sorting is not allowed."); + throw new IllegalArgumentException( + "[" + NAME + "] retriever only supports sorting by score, invalid sort criterion: " + sort.toString() + ); } } @@ -164,11 +166,8 @@ private QueryBuilder createPinnedQuery(QueryBuilder baseQuery) { } if (docs.isEmpty() == false) { return new PinnedQueryBuilder(baseQuery, docs.toArray(new SpecifiedDocument[0])); - } else if (ids.isEmpty() == false) { - return new PinnedQueryBuilder(baseQuery, ids.toArray(new String[0])); - } else { - return baseQuery; } + return new PinnedQueryBuilder(baseQuery, ids.toArray(new String[0])); } @Override From f19b718665cc04363e648e75bfd0786e3cb1e08e Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 16 Apr 2025 08:54:04 +0100 Subject: [PATCH 33/83] made unit tests robust --- .../PinnedRetrieverBuilderTests.java | 29 ++++++++++++------- .../10_pinned_retriever.yml | 8 +---- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java index b74508910b319..5e59903d8e2fe 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -152,7 +152,7 @@ public void testPinnedRetrieverParsing() throws IOException { } public void testValidation() { - expectThrows(IllegalArgumentException.class, () -> { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { new PinnedRetrieverBuilder( List.of("id1"), List.of(new SpecifiedDocument("id2", "index")), @@ -160,14 +160,17 @@ public void testValidation() { DEFAULT_RANK_WINDOW_SIZE ); }); + assertThat(e.getMessage(), equalTo("Both 'ids' and 'docs' cannot be specified at the same time")); - PinnedRetrieverBuilder builder = new PinnedRetrieverBuilder( - List.of(), - List.of(), - new TestRetrieverBuilder("test"), - DEFAULT_RANK_WINDOW_SIZE - ); - assertNotNull(builder); + e = expectThrows(IllegalArgumentException.class, () -> { + new PinnedRetrieverBuilder( + List.of(), + List.of(), + new TestRetrieverBuilder("test"), + DEFAULT_RANK_WINDOW_SIZE + ); + }); + assertThat(e.getMessage(), equalTo("Either 'ids' or 'docs' must be provided and non-empty for pinned retriever")); } public void testValidateSort() { @@ -190,12 +193,18 @@ public void testValidateSort() { SearchSourceBuilder customSortSource = new SearchSourceBuilder(); customSortSource.sort("field1"); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.finalizeSourceBuilder(customSortSource)); - assertThat(e.getMessage(), equalTo("Pinned retriever only supports sorting by score. Custom sorting is not allowed.")); + assertThat( + e.getMessage(), + equalTo("[pinned] retriever only supports sorting by score, invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}") + ); SearchSourceBuilder multipleSortsSource = new SearchSourceBuilder(); multipleSortsSource.sort("_score"); multipleSortsSource.sort("field1"); e = expectThrows(IllegalArgumentException.class, () -> builder.finalizeSourceBuilder(multipleSortsSource)); - assertThat(e.getMessage(), equalTo("Pinned retriever only supports sorting by score. Custom sorting is not allowed.")); + assertThat( + e.getMessage(), + equalTo("[pinned] retriever only supports sorting by score, invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}") + ); } } diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index 79cf6739ada4f..75d771365369a 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -320,6 +320,7 @@ setup: --- "pinned retriever with empty parameters": - do: + catch: /Either 'ids' or 'docs' must be provided and non-empty for pinned retriever/ search: index: test-index1 body: @@ -330,13 +331,6 @@ setup: query: match_all: {} - - match: { hits.total.value: 5 } - - match: { hits.hits.0._id: doc1 } - - match: { hits.hits.1._id: doc2 } - - match: { hits.hits.2._id: doc3 } - - match: { hits.hits.3._id: doc4 } - - match: { hits.hits.4._id: doc5 } - --- "pinned retriever error case - both ids and docs": - do: From 7e90b88c93ed9aa76ee719eb2394d3b927df33f8 Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 16 Apr 2025 08:55:12 +0100 Subject: [PATCH 34/83] Remove query.es from version control --- query.es | 65 -------------------------------------------------------- 1 file changed, 65 deletions(-) delete mode 100644 query.es diff --git a/query.es b/query.es deleted file mode 100644 index b2c5f970a678a..0000000000000 --- a/query.es +++ /dev/null @@ -1,65 +0,0 @@ -% Title: Elasticsearch Query with Pinned Documents - - -POST my-index/_doc/1 -{ "title": "Beginner's Guide to ML" } - -POST my-index/_doc/2 -{ "title": "Sponsored ML Course" } - -POST my-index/_doc/3 -{ "title": "Advanced Topics in Machine Learning" } - - -GET my-index/_search -{ - "query": { - "match": { - "title": "machine learning" - } - } -} - -GET my-index/_search -{ - "query": { - "pinned": { - "ids": ["1", "2"], - "organic": { - "match": { - "title": "machine learning" - } - } - } - } -} - -GET my-index/_search -{ - "query": { - "pinned": { - "ids": ["doc-id-1", "doc-id-2"], // pinned to top - "organic": { - "match": { - "title": "machine learning" - } - } - } - } -} - -GET my-index/_search -{ - "explain": true, - "query": { - "pinned": { - "ids": ["1", "2"], - "organic": { - "match": { - "title": "machine learning" - } - } - } - } -} - From 923e1543a8a9a3845acaaab118577135c2fac9b6 Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 16 Apr 2025 08:55:47 +0100 Subject: [PATCH 35/83] Remove result.json from version control --- result.json | 223 ---------------------------------------------------- 1 file changed, 223 deletions(-) delete mode 100644 result.json diff --git a/result.json b/result.json deleted file mode 100644 index 85e54ebd82d6d..0000000000000 --- a/result.json +++ /dev/null @@ -1,223 +0,0 @@ -{ - "took": 30, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": { - "value": 3, - "relation": "eq" - }, - "max_score": 1.7014124e+38, - "hits": [ - { - "_shard": "[my-index][0]", - "_node": "UbGQ3NQ0RMaG4J2rApFQPw", - "_index": "my-index", - "_id": "1", - "_score": 1.7014124e+38, - "_source": { - "title": "Beginner's Guide to ML" - }, - "_explanation": { - "value": 1.7014124e+38, - "description": "max of:", - "details": [ - { - "value": 1.7014124e+38, - "description": "max of:", - "details": [ - { - "value": 1.7014124e+38, - "description": "ConstantScore(_id:([fe 1f]))^1.7014124E38", - "details": [] - } - ] - } - ] - } - }, - { - "_shard": "[my-index][0]", - "_node": "UbGQ3NQ0RMaG4J2rApFQPw", - "_index": "my-index", - "_id": "2", - "_score": 1.7014122e+38, - "_source": { - "title": "Sponsored ML Course" - }, - "_explanation": { - "value": 1.7014122e+38, - "description": "max of:", - "details": [ - { - "value": 1.7014122e+38, - "description": "max of:", - "details": [ - { - "value": 1.7014122e+38, - "description": "ConstantScore(_id:([fe 2f]))^1.7014122E38", - "details": [] - } - ] - } - ] - } - }, - { - "_shard": "[my-index][0]", - "_node": "UbGQ3NQ0RMaG4J2rApFQPw", - "_index": "my-index", - "_id": "3", - "_score": 1.7796488, - "_source": { - "title": "Advanced Topics in Machine Learning" - }, - "_explanation": { - "value": 1.7796488, - "description": "max of:", - "details": [ - { - "value": 1.7796488, - "description": "sum of:", - "details": [ - { - "value": 0.8898244, - "description": "weight(title:machine in 2) [PerFieldSimilarity], result of:", - "details": [ - { - "value": 0.8898244, - "description": "score(freq=1.0), computed as boost * idf * tf from:", - "details": [ - { - "value": 2.2, - "description": "boost", - "details": [] - }, - { - "value": 0.98082924, - "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:", - "details": [ - { - "value": 1, - "description": "n, number of documents containing term", - "details": [] - }, - { - "value": 3, - "description": "N, total number of documents with field", - "details": [] - } - ] - }, - { - "value": 0.4123711, - "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:", - "details": [ - { - "value": 1, - "description": "freq, occurrences of term within document", - "details": [] - }, - { - "value": 1.2, - "description": "k1, term saturation parameter", - "details": [] - }, - { - "value": 0.75, - "description": "b, length normalization parameter", - "details": [] - }, - { - "value": 5, - "description": "dl, length of field", - "details": [] - }, - { - "value": 4, - "description": "avgdl, average length of field", - "details": [] - } - ] - } - ] - } - ] - }, - { - "value": 0.8898244, - "description": "weight(title:learning in 2) [PerFieldSimilarity], result of:", - "details": [ - { - "value": 0.8898244, - "description": "score(freq=1.0), computed as boost * idf * tf from:", - "details": [ - { - "value": 2.2, - "description": "boost", - "details": [] - }, - { - "value": 0.98082924, - "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:", - "details": [ - { - "value": 1, - "description": "n, number of documents containing term", - "details": [] - }, - { - "value": 3, - "description": "N, total number of documents with field", - "details": [] - } - ] - }, - { - "value": 0.4123711, - "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:", - "details": [ - { - "value": 1, - "description": "freq, occurrences of term within document", - "details": [] - }, - { - "value": 1.2, - "description": "k1, term saturation parameter", - "details": [] - }, - { - "value": 0.75, - "description": "b, length normalization parameter", - "details": [] - }, - { - "value": 5, - "description": "dl, length of field", - "details": [] - }, - { - "value": 4, - "description": "avgdl, average length of field", - "details": [] - } - ] - } - ] - } - ] - } - ] - } - ] - } - } - ] - } -} \ No newline at end of file From 583324b36aec85b9a6be62ca7be1bb4368588834 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 16 Apr 2025 08:03:01 +0000 Subject: [PATCH 36/83] [CI] Auto commit changes from spotless --- .../retriever/PinnedRetrieverBuilderTests.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java index 5e59903d8e2fe..1b272b0533d6b 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -163,12 +163,7 @@ public void testValidation() { assertThat(e.getMessage(), equalTo("Both 'ids' and 'docs' cannot be specified at the same time")); e = expectThrows(IllegalArgumentException.class, () -> { - new PinnedRetrieverBuilder( - List.of(), - List.of(), - new TestRetrieverBuilder("test"), - DEFAULT_RANK_WINDOW_SIZE - ); + new PinnedRetrieverBuilder(List.of(), List.of(), new TestRetrieverBuilder("test"), DEFAULT_RANK_WINDOW_SIZE); }); assertThat(e.getMessage(), equalTo("Either 'ids' or 'docs' must be provided and non-empty for pinned retriever")); } @@ -195,7 +190,9 @@ public void testValidateSort() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.finalizeSourceBuilder(customSortSource)); assertThat( e.getMessage(), - equalTo("[pinned] retriever only supports sorting by score, invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}") + equalTo( + "[pinned] retriever only supports sorting by score, invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}" + ) ); SearchSourceBuilder multipleSortsSource = new SearchSourceBuilder(); @@ -204,7 +201,9 @@ public void testValidateSort() { e = expectThrows(IllegalArgumentException.class, () -> builder.finalizeSourceBuilder(multipleSortsSource)); assertThat( e.getMessage(), - equalTo("[pinned] retriever only supports sorting by score, invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}") + equalTo( + "[pinned] retriever only supports sorting by score, invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}" + ) ); } } From aebe17a07770663cd0712b53b3085e4d498f9909 Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 16 Apr 2025 12:45:09 +0100 Subject: [PATCH 37/83] Everything except explanation is fixed --- .../10_pinned_retriever.yml | 98 ++++++++++++------- 1 file changed, 63 insertions(+), 35 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index 75d771365369a..743a22f07a252 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -2,10 +2,6 @@ setup: - do: indices.create: index: test-index1 - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - do: bulk: @@ -30,7 +26,9 @@ setup: --- "pinned retriever parameter variations": + - skip: { features: headers } - do: + headers: { Content-Type: application/json } search: index: test-index1 body: @@ -44,14 +42,18 @@ setup: retriever: standard: query: - match_all: {} + match: { text: "document" } - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014124E38 } - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.1._score: 1.7014122E38 } - match: { hits.hits.2._id: doc3 } + - match: { hits.hits.2._score < 100.0 } - do: + headers: { Content-Type: application/json } search: index: test-index1 body: @@ -61,13 +63,16 @@ setup: retriever: standard: query: - match_all: {} + match: { text: "document" } - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014122E38 } - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.1._score < 100.0 } - do: + headers: { Content-Type: application/json } search: index: test-index1 body: @@ -79,11 +84,13 @@ setup: retriever: standard: query: - match_all: {} + match: { text: "document" } - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014122E38 } - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.1._score < 100.0 } --- "pinned retriever dynamic pinning and ordering": @@ -101,16 +108,16 @@ setup: retriever: standard: query: - match_all: {} + match: { text: "document" } explain: true - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - - match: { hits.hits.1._id: doc2 } - - match: { hits.hits.2._id: doc3 } - match: { hits.hits.0._score: 1.7014124E38 } + - match: { hits.hits.1._id: doc2 } - match: { hits.hits.1._score: 1.7014122E38 } - - match: { hits.hits.2._score: 1.0 } + - match: { hits.hits.2._id: doc3 } + - match: { hits.hits.2._score < 100.0 } - do: headers: @@ -124,18 +131,18 @@ setup: retriever: standard: query: - match_all: {} + match: { text: "document" } explain: true - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014124E38 } - match: { hits.hits.1._id: doc3 } + - match: { hits.hits.1._score: 1.7014122E38 } - match: { hits.hits.2._id: doc2 } + - match: { hits.hits.2._score < 100.0 } - match: { hits.hits.3._id: doc4 } - match: { hits.hits.4._id: doc5 } - - match: { hits.hits.0._score: 1.7014124E38 } - - match: { hits.hits.1._score: 1.7014122E38 } - - match: { hits.hits.2._score: 1.0 } --- "pinned retriever combined with rrf": @@ -175,12 +182,14 @@ setup: - match: { hits.hits.0._id: doc1 } - match: { hits.hits.0._score: 1.7014122E38 } - match: { hits.hits.1._id: doc3 } + - match: { hits.hits.1._score < 100.0 } - match: { hits.hits.2._id: doc2 } - - match: { hits.hits.1._score < hits.hits.0._score } --- "pinned retriever with pagination": + - skip: { features: headers } - do: + headers: { Content-Type: application/json } search: index: test-index1 body: @@ -192,15 +201,18 @@ setup: retriever: standard: query: - match_all: {} + match: { text: "document" } - match: { hits.total.value: 5 } - length: { hits.hits: 1 } - match: { hits.hits.0._id: doc2 } + - match: { hits.hits.0._score: 1.7014122E38 } --- "pinned retriever as a sub-retriever": + - skip: { features: headers } - do: + headers: { Content-Type: application/json } search: index: test-index1 body: @@ -210,23 +222,24 @@ setup: - standard: query: - match_all: {} + match: { text: "document" } - pinned: ids: ["doc1", "doc2"] retriever: standard: query: - match_all: {} + match: { text: "document" } - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - - match: { hits.hits.1._id: doc2 } - - match: { hits.hits.2._id: doc3 } + - lt: { hits.hits.0._score: 100.0 } --- "pinned retriever with explicit sort on score": + - skip: { features: headers } - do: + headers: { Content-Type: application/json } search: index: test-index1 body: @@ -236,13 +249,16 @@ setup: retriever: standard: query: - match_all: {} + match: { text: "document" } sort: [ "_score" ] - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014124E38 } - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.1._score: 1.7014122E38 } - match: { hits.hits.2._id: doc3 } + - match: { hits.hits.2._score < 100.0 } --- "pinned retriever with rank window size": @@ -261,12 +277,13 @@ setup: retriever: standard: query: - match_all: {} + match: { text: "document" } rank_window_size: 1 - match: { hits.total.value: 5 } - length: { hits.hits: 1 } - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014124E38 } - do: headers: @@ -280,14 +297,17 @@ setup: retriever: standard: query: - match_all: {} + match: { text: "document" } rank_window_size: 10 - match: { hits.total.value: 5 } - length: { hits.hits: 5 } - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014124E38 } - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.1._score: 1.7014122E38 } - match: { hits.hits.2._id: doc3 } + - match: { hits.hits.2._score < 100.0 } --- "pinned retriever explanation": @@ -306,16 +326,24 @@ setup: retriever: standard: query: - match_all: {} + match: { text: "document" } explain: true - match: { hits.hits.0._id: doc1 } - - match: { hits.hits.0._explanation.value: 1.0 } - - match: { hits.hits.0._explanation.description: "doc [0] with an original score of [1.7014124E38] is at rank [1] from the following source queries." } + # - match: { hits.hits.0._explanation.value: 1.7014124E38 } + - match: + hits.hits.0._explanation.description: 'doc \[\d+\] with an original score of \[.+] is at rank \[\d+\] from the following source queries\.' + # - match: { hits.hits.0._explanation.details.0.description: "boost" } + # - match: + # hits.hits.0._explanation.details.1.description: "/doc \[\d+\] with an original score of \[.+] is at rank \[\d+\] from the following source queries\./" - match: { hits.hits.1._id: doc2 } - - match: { hits.hits.1._explanation.value: 2.0 } - - match: { hits.hits.1._explanation.description: "doc [1] with an original score of [1.7014122E38] is at rank [2] from the following source queries." } + # - match: { hits.hits.1._explanation.value: 1.7014122E38 } + - match: + hits.hits.1._explanation.description: 'doc \[\d+\] with an original score of \[.+] is at rank \[\d+\] from the following source queries\.' + # - match: { hits.hits.1._explanation.details.0.description: "boost" } + # - match: + # hits.hits.1._explanation.details.1.description: "/doc \[\d+\] with an original score of \[.+] is at rank \[\d+\] from the following source queries\./" --- "pinned retriever with empty parameters": @@ -329,7 +357,7 @@ setup: retriever: standard: query: - match_all: {} + match: { text: "document" } --- "pinned retriever error case - both ids and docs": @@ -347,7 +375,7 @@ setup: retriever: standard: query: - match_all: {} + match: { text: "document" } --- "pinned retriever error case - duplicate id": @@ -362,7 +390,7 @@ setup: retriever: standard: query: - match_all: {} + match: { text: "document" } --- "pinned retriever with ids and empty docs array": @@ -377,7 +405,7 @@ setup: retriever: standard: query: - match_all: {} + match: { text: "document" } - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - match: { hits.hits.1._id: doc2 } @@ -386,7 +414,7 @@ setup: --- "pinned retriever errors if sorting by anything other than score": - do: - catch: /Pinned retriever only supports sorting by score. Custom sorting is not allowed./ + catch: /\[pinned\] retriever only supports sorting by score/ search: index: test-index1 body: @@ -396,5 +424,5 @@ setup: retriever: standard: query: - match_all: {} + match: { text: "document" } sort: [ { "_id": "desc" } ] From 455db59143fa10b44627c904471252387209b4dc Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 16 Apr 2025 13:04:32 +0100 Subject: [PATCH 38/83] added duplicate doc test --- .../10_pinned_retriever.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index 743a22f07a252..71cd9c787099b 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -392,6 +392,25 @@ setup: query: match: { text: "document" } +--- +"pinned retriever error case - duplicate doc": + - do: + catch: /duplicate doc found/ + search: + index: test-index1 + body: + retriever: + pinned: + docs: + - _index: test-index1 + _id: doc1 + - _index: test-index1 + _id: doc1 + retriever: + standard: + query: + match: { text: "document" } + --- "pinned retriever with ids and empty docs array": - do: From c5ff33d3562c06aa52c03e31d100ff5e6e5c0f1e Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 16 Apr 2025 17:39:03 +0100 Subject: [PATCH 39/83] Applied checkstyle fix, spotless and a failing yaml --- query.es | 67 +++++ result.json | 241 ++++++++++++++++++ .../PinnedQueryBuilder.java | 2 +- .../SearchBusinessRules.java | 8 +- .../retriever/PinnedRankDoc.java | 66 ++++- .../retriever/PinnedRetrieverBuilder.java | 26 +- .../PinnedRetrieverBuilderTests.java | 6 +- ...rchBusinessRulesClientYamlTestSuiteIT.java | 6 + ...usinessRulesClientYamlTestSuiteIT.java.new | 53 ++++ .../10_pinned_retriever.yml | 18 +- 10 files changed, 470 insertions(+), 23 deletions(-) create mode 100644 query.es create mode 100644 result.json create mode 100644 x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java.new diff --git a/query.es b/query.es new file mode 100644 index 0000000000000..8e9e34cd5d2c8 --- /dev/null +++ b/query.es @@ -0,0 +1,67 @@ +% Title: Elasticsearch Query with Pinned Documents + + +POST my-index/_doc/1 +{ + "title": "Beginner's Guide to ML" +} + +POST my-index/_doc/2 +{ "title": "Sponsored ML Course" } + +POST my-index/_doc/3 +{ "title": "Advanced Topics in Machine Learning" } + + +GET my-index/_search +{ + "query": { + "match": { + "title": "machine learning" + } + } +} + +GET my-index/_search +{ + "query": { + "pinned": { + "ids": ["1", "2"], + "organic": { + "match": { + "title": "machine learning" + } + } + } + } +} + +GET my-index/_search +{ + "explain": true, + "query": { + "pinned": { + "ids": ["1", "2"], + "organic": { + "match": { + "title": "machine learning" + } + } + } + } +} + + +GET my-index/_search +{ + "query": { + "pinned": { + "ids": ["doc-id-1", "doc-id-2"], // pinned to top + "organic": { + "match": { + "title": "machine learning" + } + } + } + } +} \ No newline at end of file diff --git a/result.json b/result.json new file mode 100644 index 0000000000000..4ae26beebf34a --- /dev/null +++ b/result.json @@ -0,0 +1,241 @@ +{ + "took": 438, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 3, + "relation": "eq" + }, + "max_score": 1.7014124e+38, + "hits": [ + { + "_shard": "[my-index][0]", + "_node": "s-p9QVgHQP6qeCA-tAdB1g", + "_index": "my-index", + "_id": "1", + "_score": 1.7014124e+38, + "_source": { + "title": "Beginner's Guide to ML" + }, + "_explanation": { + "value": 1, + "description": "doc [0] with an original score of [1.7014124E38] is at rank [1] from the following source queries.", + "details": [ + { + "value": 1.7014124e+38, + "description": "max of:", + "details": [ + { + "value": 1.7014124e+38, + "description": "max of:", + "details": [ + { + "value": 1.7014124e+38, + "description": "ConstantScore(_id:([fe 1f]))^1.7014124E38", + "details": [] + } + ] + } + ] + } + ] + } + }, + { + "_shard": "[my-index][0]", + "_node": "s-p9QVgHQP6qeCA-tAdB1g", + "_index": "my-index", + "_id": "2", + "_score": 1.7014122e+38, + "_source": { + "title": "Sponsored ML Course" + }, + "_explanation": { + "value": 2, + "description": "doc [1] with an original score of [1.7014122E38] is at rank [2] from the following source queries.", + "details": [ + { + "value": 1.7014122e+38, + "description": "max of:", + "details": [ + { + "value": 1.7014122e+38, + "description": "max of:", + "details": [ + { + "value": 1.7014122e+38, + "description": "ConstantScore(_id:([fe 2f]))^1.7014122E38", + "details": [] + } + ] + } + ] + } + ] + } + }, + { + "_shard": "[my-index][0]", + "_node": "s-p9QVgHQP6qeCA-tAdB1g", + "_index": "my-index", + "_id": "3", + "_score": 1.7796488, + "_source": { + "title": "Advanced Topics in Machine Learning" + }, + "_explanation": { + "value": 3, + "description": "doc [2] with an original score of [1.7796488] is at rank [3] from the following source queries.", + "details": [ + { + "value": 1.7796488, + "description": "max of:", + "details": [ + { + "value": 1.7796488, + "description": "sum of:", + "details": [ + { + "value": 0.8898244, + "description": "weight(title:machine in 0) [PerFieldSimilarity], result of:", + "details": [ + { + "value": 0.8898244, + "description": "score(freq=1.0), computed as boost * idf * tf from:", + "details": [ + { + "value": 2.2, + "description": "boost", + "details": [] + }, + { + "value": 0.98082924, + "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:", + "details": [ + { + "value": 1, + "description": "n, number of documents containing term", + "details": [] + }, + { + "value": 3, + "description": "N, total number of documents with field", + "details": [] + } + ] + }, + { + "value": 0.4123711, + "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:", + "details": [ + { + "value": 1, + "description": "freq, occurrences of term within document", + "details": [] + }, + { + "value": 1.2, + "description": "k1, term saturation parameter", + "details": [] + }, + { + "value": 0.75, + "description": "b, length normalization parameter", + "details": [] + }, + { + "value": 5, + "description": "dl, length of field", + "details": [] + }, + { + "value": 4, + "description": "avgdl, average length of field", + "details": [] + } + ] + } + ] + } + ] + }, + { + "value": 0.8898244, + "description": "weight(title:learning in 0) [PerFieldSimilarity], result of:", + "details": [ + { + "value": 0.8898244, + "description": "score(freq=1.0), computed as boost * idf * tf from:", + "details": [ + { + "value": 2.2, + "description": "boost", + "details": [] + }, + { + "value": 0.98082924, + "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:", + "details": [ + { + "value": 1, + "description": "n, number of documents containing term", + "details": [] + }, + { + "value": 3, + "description": "N, total number of documents with field", + "details": [] + } + ] + }, + { + "value": 0.4123711, + "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:", + "details": [ + { + "value": 1, + "description": "freq, occurrences of term within document", + "details": [] + }, + { + "value": 1.2, + "description": "k1, term saturation parameter", + "details": [] + }, + { + "value": 0.75, + "description": "b, length normalization parameter", + "details": [] + }, + { + "value": 5, + "description": "dl, length of field", + "details": [] + }, + { + "value": 4, + "description": "avgdl, average length of field", + "details": [] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + } + ] + } +} \ No newline at end of file diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java index c7a3b10cd7038..4b2d48ca1eaac 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java @@ -59,7 +59,7 @@ public class PinnedQueryBuilder extends AbstractQueryBuilder // Organic queries will have their scores capped to this number range, // We reserve the highest float exponent for scores of pinned queries - private static final float MAX_ORGANIC_SCORE = Float.intBitsToFloat((0xfe << 23)) - 1; + public static final float MAX_ORGANIC_SCORE = Float.intBitsToFloat((0xfe << 23)) - 1; public PinnedQueryBuilder(QueryBuilder organicQuery, String... ids) { this(organicQuery, Arrays.asList(ids), null); diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java index 522a0e76ff89c..2ff8f445b3cb1 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -9,6 +9,8 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.plugins.SearchPlugin.QuerySpec; +import org.elasticsearch.plugins.SearchPlugin.RetrieverSpec; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder; @@ -20,7 +22,11 @@ public class SearchBusinessRules extends Plugin implements SearchPlugin { @Override public List> getQueries() { - return singletonList(new QuerySpec<>(PinnedQueryBuilder.NAME, PinnedQueryBuilder::new, PinnedQueryBuilder::fromXContent)); + // Assuming PinnedQueryBuilder exists and has NAME, constructor, and fromXContent + // If PinnedQueryBuilder was removed or changed, this needs adjustment. + // return singletonList(new QuerySpec<>(PinnedQueryBuilder.NAME, PinnedQueryBuilder::new, PinnedQueryBuilder::fromXContent)); + // For now, return empty list if PinnedQueryBuilder is not ready/defined + return List.of(); } @Override diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java index 9f4dcbaa6cc78..bb717d6fe2be0 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java @@ -7,16 +7,33 @@ package org.elasticsearch.xpack.searchbusinessrules.retriever; +import org.apache.lucene.search.Explanation; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.rank.RankDoc; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; public class PinnedRankDoc extends RankDoc { + public static final String NAME = "pinned_rank_doc"; + private final boolean isPinned; private final String pinnedBy; public PinnedRankDoc(int docId, float score, int shardIndex, boolean isPinned, String pinnedBy) { super(docId, score, shardIndex); this.isPinned = isPinned; - this.pinnedBy = pinnedBy; + this.pinnedBy = (isPinned) ? Objects.requireNonNull(pinnedBy, "pinnedBy cannot be null when isPinned is true") : null; + } + + public PinnedRankDoc(StreamInput in) throws IOException { + super(in); + this.isPinned = in.readBoolean(); + this.pinnedBy = in.readOptionalString(); } public boolean isPinned() { @@ -27,8 +44,55 @@ public String getPinnedBy() { return pinnedBy; } + @Override + public Explanation explain(Explanation[] sources, String[] queryNames) { + if (isPinned) { + return Explanation.match(score, "Pinned document by " + pinnedBy + ", original explanation:", sources); + } else { + return super.explain(sources, queryNames); + } + } + @Override public String toString() { return super.toString() + ", isPinned=" + isPinned + ", pinnedBy=" + pinnedBy; } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeBoolean(isPinned); + out.writeOptionalString(pinnedBy); + } + + @Override + protected boolean doEquals(RankDoc rd) { + if (rd instanceof PinnedRankDoc other) { + return this.isPinned == other.isPinned && Objects.equals(this.pinnedBy, other.pinnedBy); + } else { + return false; + } + } + + @Override + protected int doHashCode() { + return Objects.hash(super.doHashCode(), isPinned, pinnedBy); + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.V_8_0_0; + } + + @Override + protected void doToXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("is_pinned", isPinned); + if (pinnedBy != null) { + builder.field("pinned_by", pinnedBy); + } + } } diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index 1aeba640a0223..c2e2193d476d6 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -12,6 +12,7 @@ import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RankDocsQueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.rank.RankDoc; import org.elasticsearch.search.retriever.CompoundRetrieverBuilder; @@ -24,6 +25,7 @@ import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContent.Params; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder; @@ -205,10 +207,19 @@ protected RankDoc[] combineInnerRetrieverResults(List rankResults, b RankDoc[] rankDocs = new RankDoc[scoreDocs.length]; for (int i = 0; i < scoreDocs.length; i++) { ScoreDoc scoreDoc = scoreDocs[i]; - boolean isPinned = docs.stream().anyMatch(doc -> doc.id().equals(String.valueOf(scoreDoc.doc))) - || ids.contains(String.valueOf(scoreDoc.doc)); - String pinnedBy = docs.stream().anyMatch(doc -> doc.id().equals(String.valueOf(scoreDoc.doc))) ? "docs" : "ids"; - rankDocs[i] = new PinnedRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex, isPinned, pinnedBy); + + if (explain) { + boolean isPinned = scoreDoc.score > PinnedQueryBuilder.MAX_ORGANIC_SCORE; + if (isPinned) { + String pinnedBy = (this.ids != null && this.ids.isEmpty() == false) ? "ids" : "docs"; + rankDocs[i] = new PinnedRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex, true, pinnedBy); + } else { + rankDocs[i] = new RankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex); + } + } else { + rankDocs[i] = new RankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex); + } + rankDocs[i].rank = i + 1; } return rankDocs; @@ -246,8 +257,11 @@ public QueryBuilder topDocsQuery() { @Override public QueryBuilder explainQuery() { - QueryBuilder baseQuery = in.explainQuery(); - return createPinnedQuery(baseQuery); + RankDoc[] currentRankDocs = in.getRankDocs(); + if (currentRankDocs == null) { + return in.explainQuery(); + } + return new RankDocsQueryBuilder(currentRankDocs, new QueryBuilder[] { in.explainQuery() }, true); } } } diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java index 1b272b0533d6b..3eaf6bce8db6a 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -191,7 +191,8 @@ public void testValidateSort() { assertThat( e.getMessage(), equalTo( - "[pinned] retriever only supports sorting by score, invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}" + "[pinned] retriever only supports sorting by score, " + + "invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}" ) ); @@ -202,7 +203,8 @@ public void testValidateSort() { assertThat( e.getMessage(), equalTo( - "[pinned] retriever only supports sorting by score, invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}" + "[pinned] retriever only supports sorting by score, " + + "invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}" ) ); } diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java index e12a55c6fe767..13fa6814f1939 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java @@ -1,3 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ package org.elasticsearch.xpack.searchbusinessrules; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java.new b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java.new new file mode 100644 index 0000000000000..948f8cc09d0f9 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java.new @@ -0,0 +1,53 @@ +package org.elasticsearch.xpack.searchbusinessrules; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; + +public class SearchBusinessRulesClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.security.enabled", "false") + .build(); + + public SearchBusinessRulesClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index 71cd9c787099b..32827eebe2d65 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -311,11 +311,11 @@ setup: --- "pinned retriever explanation": - - skip: - features: headers + - skip: { features: headers } - do: headers: + # Force JSON content type so that we use a parser that interprets the floating-point score as a double Content-Type: application/json search: index: test-index1 @@ -330,20 +330,14 @@ setup: explain: true - match: { hits.hits.0._id: doc1 } - # - match: { hits.hits.0._explanation.value: 1.7014124E38 } + - gt: { hits.hits.0._score: 1.0 } - match: - hits.hits.0._explanation.description: 'doc \[\d+\] with an original score of \[.+] is at rank \[\d+\] from the following source queries\.' - # - match: { hits.hits.0._explanation.details.0.description: "boost" } - # - match: - # hits.hits.0._explanation.details.1.description: "/doc \[\d+\] with an original score of \[.+] is at rank \[\d+\] from the following source queries\./" + hits.hits.0._explanation.description: "Pinned document by ids, original explanation:" - match: { hits.hits.1._id: doc2 } - # - match: { hits.hits.1._explanation.value: 1.7014122E38 } + - gt: { hits.hits.1._score: 1.0 } - match: - hits.hits.1._explanation.description: 'doc \[\d+\] with an original score of \[.+] is at rank \[\d+\] from the following source queries\.' - # - match: { hits.hits.1._explanation.details.0.description: "boost" } - # - match: - # hits.hits.1._explanation.details.1.description: "/doc \[\d+\] with an original score of \[.+] is at rank \[\d+\] from the following source queries\./" + hits.hits.1._explanation.description: "Pinned document by ids, original explanation:" --- "pinned retriever with empty parameters": From ba40533033a8f064e42eaa72469a761bd364e605 Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 16 Apr 2025 18:42:04 +0100 Subject: [PATCH 40/83] Improvements based on PR comments --- .../retriever/PinnedRetrieverBuilder.java | 8 ++++---- .../retriever/PinnedRetrieverBuilderTests.java | 1 + .../10_pinned_retriever.yml | 17 +++++++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index c2e2193d476d6..6673dfbc51d95 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -160,12 +160,12 @@ public int rankWindowSize() { * Prioritizes docs over ids if both are present. * * @param baseQuery the base query to pin documents to - * @return a PinnedQueryBuilder or the original query if no pinned documents + * @return a PinnedQueryBuilder + * @throws IllegalArgumentException if baseQuery is null */ private QueryBuilder createPinnedQuery(QueryBuilder baseQuery) { - if (baseQuery == null) { - baseQuery = new MatchAllQueryBuilder(); - } + Objects.requireNonNull(baseQuery, "Underlying query cannot be null for pinned retriever"); + if (docs.isEmpty() == false) { return new PinnedQueryBuilder(baseQuery, docs.toArray(new SpecifiedDocument[0])); } diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java index 3eaf6bce8db6a..7f991fc2f29c5 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.searchbusinessrules.SpecifiedDocument; + import java.io.IOException; import java.util.ArrayList; import java.util.List; diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index 32827eebe2d65..fe33905c81f7d 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -333,11 +333,15 @@ setup: - gt: { hits.hits.0._score: 1.0 } - match: hits.hits.0._explanation.description: "Pinned document by ids, original explanation:" + - match: + hits.hits.0._explanation.details.0.description: "doc [0] with an original score of [1.7014124E38] is at rank [1] from the following source queries." - match: { hits.hits.1._id: doc2 } - gt: { hits.hits.1._score: 1.0 } - match: hits.hits.1._explanation.description: "Pinned document by ids, original explanation:" + - match: + hits.hits.1._explanation.details.0.description: "doc [1] with an original score of [1.7014122E38] is at rank [2] from the following source queries." --- "pinned retriever with empty parameters": @@ -439,3 +443,16 @@ setup: query: match: { text: "document" } sort: [ { "_id": "desc" } ] + +--- +"pinned retriever error case - null inner query": + - do: + catch: /Underlying query cannot be null for pinned retriever/ + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1"] + retriever: + standard: {} From 4456504e396748c34a4eb41e7a742a10a96dec57 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 16 Apr 2025 17:48:55 +0000 Subject: [PATCH 41/83] [CI] Auto commit changes from spotless --- .../searchbusinessrules/retriever/PinnedRetrieverBuilder.java | 1 - .../retriever/PinnedRetrieverBuilderTests.java | 1 - 2 files changed, 2 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index 6673dfbc51d95..2fafd0b1fcd9b 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -10,7 +10,6 @@ import org.apache.lucene.search.ScoreDoc; import org.elasticsearch.common.ParsingException; import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.RankDocsQueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java index 7f991fc2f29c5..3eaf6bce8db6a 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.searchbusinessrules.SpecifiedDocument; - import java.io.IOException; import java.util.ArrayList; import java.util.List; From 3c864163c99307dee03202f46239ffbb68999672 Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 16 Apr 2025 21:37:44 +0100 Subject: [PATCH 42/83] Modified the unit test to accomodate the change in createPinnedQuery --- .../retriever/PinnedRetrieverBuilderTests.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java index 3eaf6bce8db6a..528d1226f04d8 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -9,6 +9,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.retriever.RetrieverBuilder; @@ -169,6 +171,7 @@ public void testValidation() { } public void testValidateSort() { + PinnedRetrieverBuilder builder = new PinnedRetrieverBuilder( List.of("id1"), List.of(), @@ -176,35 +179,39 @@ public void testValidateSort() { DEFAULT_RANK_WINDOW_SIZE ); + QueryBuilder dummyQuery = new MatchAllQueryBuilder(); + SearchSourceBuilder emptySource = new SearchSourceBuilder(); + emptySource.query(dummyQuery); builder.finalizeSourceBuilder(emptySource); assertThat(emptySource.sorts(), equalTo(null)); SearchSourceBuilder scoreSource = new SearchSourceBuilder(); + scoreSource.query(dummyQuery); scoreSource.sort("_score"); builder.finalizeSourceBuilder(scoreSource); assertThat(scoreSource.sorts().size(), equalTo(1)); SearchSourceBuilder customSortSource = new SearchSourceBuilder(); + customSortSource.query(dummyQuery); customSortSource.sort("field1"); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.finalizeSourceBuilder(customSortSource)); assertThat( e.getMessage(), equalTo( - "[pinned] retriever only supports sorting by score, " - + "invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}" + "[" + PinnedRetrieverBuilder.NAME + "] retriever only supports sorting by score, invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}" ) ); SearchSourceBuilder multipleSortsSource = new SearchSourceBuilder(); + multipleSortsSource.query(dummyQuery); multipleSortsSource.sort("_score"); multipleSortsSource.sort("field1"); e = expectThrows(IllegalArgumentException.class, () -> builder.finalizeSourceBuilder(multipleSortsSource)); assertThat( e.getMessage(), equalTo( - "[pinned] retriever only supports sorting by score, " - + "invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}" + "[" + PinnedRetrieverBuilder.NAME + "] retriever only supports sorting by score, invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}" ) ); } From f2035a6cc1c7a03e2c408eb44970f25e0ea387ab Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 16 Apr 2025 22:40:28 +0100 Subject: [PATCH 43/83] Split the yaml to test for basic and trial, cleanedup and acted on all the comments --- .../PinnedRetrieverBuilderTests.java | 10 +- ...rchBusinessRulesClientYamlTestSuiteIT.java | 4 +- ...BusinessRulesRrfClientYamlTestSuiteIT.java | 44 +++++++++ ...ever.yml => 10_pinned_retriever_basic.yml} | 68 -------------- .../11_pinned_retriever_rrf.yml | 94 +++++++++++++++++++ 5 files changed, 148 insertions(+), 72 deletions(-) create mode 100644 x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesRrfClientYamlTestSuiteIT.java rename x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/{10_pinned_retriever.yml => 10_pinned_retriever_basic.yml} (85%) create mode 100644 x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/11_pinned_retriever_rrf.yml diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java index 528d1226f04d8..63d5ef381deed 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -199,7 +199,10 @@ public void testValidateSort() { assertThat( e.getMessage(), equalTo( - "[" + PinnedRetrieverBuilder.NAME + "] retriever only supports sorting by score, invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}" + "[" + + PinnedRetrieverBuilder.NAME + + "] retriever only supports sorting by score, " + + "invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}" ) ); @@ -211,7 +214,10 @@ public void testValidateSort() { assertThat( e.getMessage(), equalTo( - "[" + PinnedRetrieverBuilder.NAME + "] retriever only supports sorting by score, invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}" + "[" + + PinnedRetrieverBuilder.NAME + + "] retriever only supports sorting by score, " + + "invalid sort criterion: {\n \"field1\" : {\n \"order\" : \"asc\"\n }\n}" ) ); } diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java index 13fa6814f1939..43b63ceed4193 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java @@ -19,7 +19,7 @@ public class SearchBusinessRulesClientYamlTestSuiteIT extends ESClientYamlSuiteT @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) - .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.license.self_generated.type", "basic") .setting("xpack.security.enabled", "false") .build(); @@ -29,7 +29,7 @@ public SearchBusinessRulesClientYamlTestSuiteIT(final ClientYamlTestCandidate te @ParametersFactory public static Iterable parameters() throws Exception { - return ESClientYamlSuiteTestCase.createParameters(); + return ESClientYamlSuiteTestCase.createParameters(new String[] { "search-business-rules/10_pinned_retriever_basic" }); } @Override diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesRrfClientYamlTestSuiteIT.java b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesRrfClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..2eb2ba645b6c3 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesRrfClientYamlTestSuiteIT.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.searchbusinessrules; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; + +/** + * Test suite for running YAML tests specific to Pinned Retriever + RRF combination, + * requiring a trial license. + */ +public class SearchBusinessRulesRrfClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + // RRF requires trial or platinum license + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.security.enabled", "false") + .build(); + + public SearchBusinessRulesRrfClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(new String[] { "search-business-rules/11_pinned_retriever_rrf" }); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever_basic.yml similarity index 85% rename from x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml rename to x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever_basic.yml index fe33905c81f7d..300f0c03d1556 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever_basic.yml @@ -144,47 +144,6 @@ setup: - match: { hits.hits.3._id: doc4 } - match: { hits.hits.4._id: doc5 } ---- -"pinned retriever combined with rrf": - - skip: - features: headers - - - do: - headers: - Content-Type: application/json - search: - index: test-index1 - body: - retriever: - pinned: - ids: ["doc1"] - retriever: - rrf: - retrievers: [ - { - standard: { - query: { - term: { text: "document" } - } - } - }, - { - standard: { - query: { - term: { text: "three" } - } - } - } - ] - rank_window_size: 10 - - - match: { hits.total.value: 5 } - - match: { hits.hits.0._id: doc1 } - - match: { hits.hits.0._score: 1.7014122E38 } - - match: { hits.hits.1._id: doc3 } - - match: { hits.hits.1._score < 100.0 } - - match: { hits.hits.2._id: doc2 } - --- "pinned retriever with pagination": - skip: { features: headers } @@ -208,33 +167,6 @@ setup: - match: { hits.hits.0._id: doc2 } - match: { hits.hits.0._score: 1.7014122E38 } ---- -"pinned retriever as a sub-retriever": - - skip: { features: headers } - - do: - headers: { Content-Type: application/json } - search: - index: test-index1 - body: - retriever: - rrf: - retrievers: - - - standard: - query: - match: { text: "document" } - - - pinned: - ids: ["doc1", "doc2"] - retriever: - standard: - query: - match: { text: "document" } - - - match: { hits.total.value: 5 } - - match: { hits.hits.0._id: doc1 } - - lt: { hits.hits.0._score: 100.0 } - --- "pinned retriever with explicit sort on score": - skip: { features: headers } diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/11_pinned_retriever_rrf.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/11_pinned_retriever_rrf.yml new file mode 100644 index 0000000000000..e58e3fa972ac1 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/11_pinned_retriever_rrf.yml @@ -0,0 +1,94 @@ +setup: + - do: + indices.create: + index: test-index1 + + - do: + bulk: + refresh: true + index: test-index1 + body: + - index: + _id: doc1 + - { "text": "document one" } + - index: + _id: doc2 + - { "text": "document two" } + - index: + _id: doc3 + - { "text": "document three" } + - index: + _id: doc4 + - { "text": "document four" } + - index: + _id: doc5 + - { "text": "document five" } + +--- +"pinned retriever combined with rrf": + - skip: + features: headers + + - do: + headers: + Content-Type: application/json + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1"] + retriever: + rrf: + retrievers: [ + { + standard: { + query: { + term: { text: "document" } + } + } + }, + { + standard: { + query: { + term: { text: "three" } + } + } + } + ] + rank_window_size: 10 + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014122E38 } + - match: { hits.hits.1._id: doc3 } + - match: { hits.hits.1._score < 100.0 } + - match: { hits.hits.2._id: doc2 } + +--- +"pinned retriever as a sub-retriever": + - skip: { features: headers } + - do: + headers: { Content-Type: application/json } + search: + index: test-index1 + body: + retriever: + rrf: + retrievers: + - + standard: + query: + match: { text: "document" } + - + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - lt: { hits.hits.0._score: 100.0 } + From 516ed7c4e0f7e99b4e75bf52f2e60ba3d4c8dd9e Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 16 Apr 2025 22:46:34 +0100 Subject: [PATCH 44/83] Remove result.json and query.es from version control --- query.es | 67 --------------- result.json | 241 ---------------------------------------------------- 2 files changed, 308 deletions(-) delete mode 100644 query.es delete mode 100644 result.json diff --git a/query.es b/query.es deleted file mode 100644 index 8e9e34cd5d2c8..0000000000000 --- a/query.es +++ /dev/null @@ -1,67 +0,0 @@ -% Title: Elasticsearch Query with Pinned Documents - - -POST my-index/_doc/1 -{ - "title": "Beginner's Guide to ML" -} - -POST my-index/_doc/2 -{ "title": "Sponsored ML Course" } - -POST my-index/_doc/3 -{ "title": "Advanced Topics in Machine Learning" } - - -GET my-index/_search -{ - "query": { - "match": { - "title": "machine learning" - } - } -} - -GET my-index/_search -{ - "query": { - "pinned": { - "ids": ["1", "2"], - "organic": { - "match": { - "title": "machine learning" - } - } - } - } -} - -GET my-index/_search -{ - "explain": true, - "query": { - "pinned": { - "ids": ["1", "2"], - "organic": { - "match": { - "title": "machine learning" - } - } - } - } -} - - -GET my-index/_search -{ - "query": { - "pinned": { - "ids": ["doc-id-1", "doc-id-2"], // pinned to top - "organic": { - "match": { - "title": "machine learning" - } - } - } - } -} \ No newline at end of file diff --git a/result.json b/result.json deleted file mode 100644 index 4ae26beebf34a..0000000000000 --- a/result.json +++ /dev/null @@ -1,241 +0,0 @@ -{ - "took": 438, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": { - "value": 3, - "relation": "eq" - }, - "max_score": 1.7014124e+38, - "hits": [ - { - "_shard": "[my-index][0]", - "_node": "s-p9QVgHQP6qeCA-tAdB1g", - "_index": "my-index", - "_id": "1", - "_score": 1.7014124e+38, - "_source": { - "title": "Beginner's Guide to ML" - }, - "_explanation": { - "value": 1, - "description": "doc [0] with an original score of [1.7014124E38] is at rank [1] from the following source queries.", - "details": [ - { - "value": 1.7014124e+38, - "description": "max of:", - "details": [ - { - "value": 1.7014124e+38, - "description": "max of:", - "details": [ - { - "value": 1.7014124e+38, - "description": "ConstantScore(_id:([fe 1f]))^1.7014124E38", - "details": [] - } - ] - } - ] - } - ] - } - }, - { - "_shard": "[my-index][0]", - "_node": "s-p9QVgHQP6qeCA-tAdB1g", - "_index": "my-index", - "_id": "2", - "_score": 1.7014122e+38, - "_source": { - "title": "Sponsored ML Course" - }, - "_explanation": { - "value": 2, - "description": "doc [1] with an original score of [1.7014122E38] is at rank [2] from the following source queries.", - "details": [ - { - "value": 1.7014122e+38, - "description": "max of:", - "details": [ - { - "value": 1.7014122e+38, - "description": "max of:", - "details": [ - { - "value": 1.7014122e+38, - "description": "ConstantScore(_id:([fe 2f]))^1.7014122E38", - "details": [] - } - ] - } - ] - } - ] - } - }, - { - "_shard": "[my-index][0]", - "_node": "s-p9QVgHQP6qeCA-tAdB1g", - "_index": "my-index", - "_id": "3", - "_score": 1.7796488, - "_source": { - "title": "Advanced Topics in Machine Learning" - }, - "_explanation": { - "value": 3, - "description": "doc [2] with an original score of [1.7796488] is at rank [3] from the following source queries.", - "details": [ - { - "value": 1.7796488, - "description": "max of:", - "details": [ - { - "value": 1.7796488, - "description": "sum of:", - "details": [ - { - "value": 0.8898244, - "description": "weight(title:machine in 0) [PerFieldSimilarity], result of:", - "details": [ - { - "value": 0.8898244, - "description": "score(freq=1.0), computed as boost * idf * tf from:", - "details": [ - { - "value": 2.2, - "description": "boost", - "details": [] - }, - { - "value": 0.98082924, - "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:", - "details": [ - { - "value": 1, - "description": "n, number of documents containing term", - "details": [] - }, - { - "value": 3, - "description": "N, total number of documents with field", - "details": [] - } - ] - }, - { - "value": 0.4123711, - "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:", - "details": [ - { - "value": 1, - "description": "freq, occurrences of term within document", - "details": [] - }, - { - "value": 1.2, - "description": "k1, term saturation parameter", - "details": [] - }, - { - "value": 0.75, - "description": "b, length normalization parameter", - "details": [] - }, - { - "value": 5, - "description": "dl, length of field", - "details": [] - }, - { - "value": 4, - "description": "avgdl, average length of field", - "details": [] - } - ] - } - ] - } - ] - }, - { - "value": 0.8898244, - "description": "weight(title:learning in 0) [PerFieldSimilarity], result of:", - "details": [ - { - "value": 0.8898244, - "description": "score(freq=1.0), computed as boost * idf * tf from:", - "details": [ - { - "value": 2.2, - "description": "boost", - "details": [] - }, - { - "value": 0.98082924, - "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:", - "details": [ - { - "value": 1, - "description": "n, number of documents containing term", - "details": [] - }, - { - "value": 3, - "description": "N, total number of documents with field", - "details": [] - } - ] - }, - { - "value": 0.4123711, - "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:", - "details": [ - { - "value": 1, - "description": "freq, occurrences of term within document", - "details": [] - }, - { - "value": 1.2, - "description": "k1, term saturation parameter", - "details": [] - }, - { - "value": 0.75, - "description": "b, length normalization parameter", - "details": [] - }, - { - "value": 5, - "description": "dl, length of field", - "details": [] - }, - { - "value": 4, - "description": "avgdl, average length of field", - "details": [] - } - ] - } - ] - } - ] - } - ] - } - ] - } - ] - } - } - ] - } -} \ No newline at end of file From 97c337611ba987858bd13c8a9e38cf1b772a7934 Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 16 Apr 2025 22:49:59 +0100 Subject: [PATCH 45/83] Removed unnecessary comments --- .../xpack/searchbusinessrules/SearchBusinessRules.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java index 2ff8f445b3cb1..fd85a36c572ae 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -22,10 +22,6 @@ public class SearchBusinessRules extends Plugin implements SearchPlugin { @Override public List> getQueries() { - // Assuming PinnedQueryBuilder exists and has NAME, constructor, and fromXContent - // If PinnedQueryBuilder was removed or changed, this needs adjustment. - // return singletonList(new QuerySpec<>(PinnedQueryBuilder.NAME, PinnedQueryBuilder::new, PinnedQueryBuilder::fromXContent)); - // For now, return empty list if PinnedQueryBuilder is not ready/defined return List.of(); } From ebbf1af37d25bc3b3ca5343d5dc5d00d55dbec4c Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 16 Apr 2025 22:53:44 +0100 Subject: [PATCH 46/83] Removed redundant file --- ...usinessRulesClientYamlTestSuiteIT.java.new | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java.new diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java.new b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java.new deleted file mode 100644 index 948f8cc09d0f9..0000000000000 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java.new +++ /dev/null @@ -1,53 +0,0 @@ -package org.elasticsearch.xpack.searchbusinessrules; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.cluster.local.distribution.DistributionType; -import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.junit.ClassRule; - -public class SearchBusinessRulesClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { - - @ClassRule - public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - .setting("xpack.license.self_generated.type", "trial") - .setting("xpack.security.enabled", "false") - .build(); - - public SearchBusinessRulesClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { - super(testCandidate); - } - - @ParametersFactory - public static Iterable parameters() throws Exception { - return ESClientYamlSuiteTestCase.createParameters(); - } - - @Override - protected String getTestRestCluster() { - return cluster.getHttpAddresses(); - } -} From 9b30e8714f287fbbcf004db4d317851f1129c64b Mon Sep 17 00:00:00 2001 From: Mridula Date: Thu, 17 Apr 2025 12:34:54 +0100 Subject: [PATCH 47/83] Fixing CI build error --- plugins/build.gradle | 42 ++++++++++++++----- .../plugin/search-business-rules/build.gradle | 4 +- .../SearchBusinessRules.java | 3 +- .../SearchBusinessRulesFeatures.java | 4 +- .../10_pinned_retriever_basic.yml | 3 ++ .../11_pinned_retriever_rrf.yml | 3 ++ 6 files changed, 44 insertions(+), 15 deletions(-) diff --git a/plugins/build.gradle b/plugins/build.gradle index 90fc924d7f9d6..f38525e46d51c 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -11,17 +11,37 @@ configurations { allPlugins } -// only configure immediate children of plugins dir -configure(subprojects.findAll { it.parent.path == project.path }) { - group = 'org.elasticsearch.plugin' - apply plugin: 'elasticsearch.internal-es-plugin' +def pluginDirs = [ + rootProject.file('plugins'), + rootProject.file('x-pack/plugin') +] - esplugin { - // for local ES plugins, the name of the plugin is the same as the directory - name = project.name - licenseFile = layout.settingsDirectory.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile - noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile - } +pluginDirs.each { dir -> + if (dir.isDirectory()) { + dir.listFiles()?.each { subDir -> + if (subDir.isDirectory()) { + def proj = rootProject.findProject(":${dir.name}:${subDir.name}") + + if (proj != null) { + proj.configure(proj) { + group = 'org.elasticsearch.plugin' + apply plugin: 'elasticsearch.internal-es-plugin' - parent.artifacts.add('allPlugins', tasks.named('bundlePlugin')) + esplugin { + name = proj.name + if (dir.name == 'x-pack') { + licenseFile = rootProject.layout.settingsDirectory.file('licenses/ELASTIC-LICENSE-2.0.txt').asFile + noticeFile = rootProject.file('x-pack/NOTICE.txt') + } else { + licenseFile = rootProject.layout.settingsDirectory.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile + noticeFile = rootProject.layout.settingsDirectory.file('NOTICE.txt').asFile + } + } + + rootProject.project(':plugins').artifacts.add('allPlugins', proj.tasks.named('bundlePlugin')) + } + } + } + } + } } diff --git a/x-pack/plugin/search-business-rules/build.gradle b/x-pack/plugin/search-business-rules/build.gradle index ca9cd6f60517a..ae591c0b30085 100644 --- a/x-pack/plugin/search-business-rules/build.gradle +++ b/x-pack/plugin/search-business-rules/build.gradle @@ -14,6 +14,8 @@ base { dependencies { compileOnly project(path: xpackModule('core')) + compileOnly project(':server') + compileOnly project(':libs:x-content') testImplementation(testArtifact(project(xpackModule('core')))) testImplementation(testArtifact(project(':server'))) } @@ -21,7 +23,7 @@ dependencies { tasks.named("yamlRestTest") { usesDefaultDistribution("uses search business rules plugin") } - artifacts { restXpackTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test")) } + diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java index fd85a36c572ae..a5551f0e4866f 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -12,6 +12,7 @@ import org.elasticsearch.plugins.SearchPlugin.QuerySpec; import org.elasticsearch.plugins.SearchPlugin.RetrieverSpec; import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder; import org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder; import java.util.List; @@ -22,7 +23,7 @@ public class SearchBusinessRules extends Plugin implements SearchPlugin { @Override public List> getQueries() { - return List.of(); + return List.of(new QuerySpec<>(PinnedQueryBuilder.NAME, PinnedQueryBuilder::new, PinnedQueryBuilder::fromXContent)); } @Override diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java index d702a0fcc6ea0..af3d8375a8c93 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java @@ -18,11 +18,11 @@ public class SearchBusinessRulesFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of(); + return Set.of(PINNED_RETRIEVER_FEATURE); } @Override public Set getTestFeatures() { - return Set.of(PINNED_RETRIEVER_FEATURE); + return Set.of(); } } diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever_basic.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever_basic.yml index 300f0c03d1556..2e396c6fe7c60 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever_basic.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever_basic.yml @@ -1,4 +1,7 @@ setup: + - requires: + cluster_features: 'pinned_retriever_supported' + reason: 'test requires pinned retriever implementation' - do: indices.create: index: test-index1 diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/11_pinned_retriever_rrf.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/11_pinned_retriever_rrf.yml index e58e3fa972ac1..fd7049b96c141 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/11_pinned_retriever_rrf.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/11_pinned_retriever_rrf.yml @@ -1,4 +1,7 @@ setup: + - requires: + cluster_features: 'pinned_retriever_supported' + reason: 'test requires pinned retriever implementation' - do: indices.create: index: test-index1 From 374b37d0d5520022634b9d8908d9fc5ddb23fb2a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 17 Apr 2025 12:04:19 +0000 Subject: [PATCH 48/83] [CI] Auto commit changes from spotless --- .../xpack/searchbusinessrules/SearchBusinessRules.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java index a5551f0e4866f..a6e990e69239b 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -12,7 +12,6 @@ import org.elasticsearch.plugins.SearchPlugin.QuerySpec; import org.elasticsearch.plugins.SearchPlugin.RetrieverSpec; import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder; import org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder; import java.util.List; From 42dc070fc5ff5fd8d4fc0383179687b98aa19a4e Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 23 Apr 2025 02:08:35 +0100 Subject: [PATCH 49/83] Removed pinnedBy as it wasnt necessary --- .../retriever/PinnedRankDoc.java | 21 +++++-------------- .../retriever/PinnedRetrieverBuilder.java | 2 +- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java index bb717d6fe2be0..8549f1d91450f 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java @@ -22,32 +22,25 @@ public class PinnedRankDoc extends RankDoc { public static final String NAME = "pinned_rank_doc"; private final boolean isPinned; - private final String pinnedBy; - public PinnedRankDoc(int docId, float score, int shardIndex, boolean isPinned, String pinnedBy) { + public PinnedRankDoc(int docId, float score, int shardIndex, boolean isPinned) { super(docId, score, shardIndex); this.isPinned = isPinned; - this.pinnedBy = (isPinned) ? Objects.requireNonNull(pinnedBy, "pinnedBy cannot be null when isPinned is true") : null; } public PinnedRankDoc(StreamInput in) throws IOException { super(in); this.isPinned = in.readBoolean(); - this.pinnedBy = in.readOptionalString(); } public boolean isPinned() { return isPinned; } - public String getPinnedBy() { - return pinnedBy; - } - @Override public Explanation explain(Explanation[] sources, String[] queryNames) { if (isPinned) { - return Explanation.match(score, "Pinned document by " + pinnedBy + ", original explanation:", sources); + return Explanation.match(score, "Pinned document, original explanation:", sources); } else { return super.explain(sources, queryNames); } @@ -55,7 +48,7 @@ public Explanation explain(Explanation[] sources, String[] queryNames) { @Override public String toString() { - return super.toString() + ", isPinned=" + isPinned + ", pinnedBy=" + pinnedBy; + return super.toString() + ", isPinned=" + isPinned; } @Override @@ -66,13 +59,12 @@ public String getWriteableName() { @Override protected void doWriteTo(StreamOutput out) throws IOException { out.writeBoolean(isPinned); - out.writeOptionalString(pinnedBy); } @Override protected boolean doEquals(RankDoc rd) { if (rd instanceof PinnedRankDoc other) { - return this.isPinned == other.isPinned && Objects.equals(this.pinnedBy, other.pinnedBy); + return this.isPinned == other.isPinned; } else { return false; } @@ -80,7 +72,7 @@ protected boolean doEquals(RankDoc rd) { @Override protected int doHashCode() { - return Objects.hash(super.doHashCode(), isPinned, pinnedBy); + return Objects.hash(super.doHashCode(), isPinned); } @Override @@ -91,8 +83,5 @@ public TransportVersion getMinimalSupportedVersion() { @Override protected void doToXContent(XContentBuilder builder, Params params) throws IOException { builder.field("is_pinned", isPinned); - if (pinnedBy != null) { - builder.field("pinned_by", pinnedBy); - } } } diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index 2fafd0b1fcd9b..654fdbeba4c2b 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -211,7 +211,7 @@ protected RankDoc[] combineInnerRetrieverResults(List rankResults, b boolean isPinned = scoreDoc.score > PinnedQueryBuilder.MAX_ORGANIC_SCORE; if (isPinned) { String pinnedBy = (this.ids != null && this.ids.isEmpty() == false) ? "ids" : "docs"; - rankDocs[i] = new PinnedRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex, true, pinnedBy); + rankDocs[i] = new PinnedRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex, true); } else { rankDocs[i] = new RankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex); } From d869c9e94f64a2b969555cee73cc21d6302f9989 Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 23 Apr 2025 02:13:17 +0100 Subject: [PATCH 50/83] Removed unnecessary ToXContent Information --- .../xpack/searchbusinessrules/retriever/PinnedRankDoc.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java index 8549f1d91450f..423a23dcdbe6c 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java @@ -77,11 +77,6 @@ protected int doHashCode() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_8_0_0; - } - - @Override - protected void doToXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("is_pinned", isPinned); + return TransportVersions.V_9_1_0; } } From d67b2912e29b20af589d5755947356c654d371c9 Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 23 Apr 2025 02:41:00 +0100 Subject: [PATCH 51/83] Fixed transport version charges and cleaned up null checks: " --- .../searchbusinessrules/retriever/PinnedRankDoc.java | 2 +- .../retriever/PinnedRetrieverBuilder.java | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java index 423a23dcdbe6c..6c1be44703efd 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java @@ -77,6 +77,6 @@ protected int doHashCode() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_9_1_0; + return TransportVersions.INITIAL_ELASTICSEARCH_9_0_1; } } diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index 654fdbeba4c2b..04e83c8c25db4 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -51,7 +51,7 @@ public final class PinnedRetrieverBuilder extends CompoundRetrieverBuilder PARSER = new ConstructingObjectParser<>( @@ -124,8 +124,8 @@ private void validateSort(SearchSourceBuilder source) { public PinnedRetrieverBuilder(List ids, List docs, RetrieverBuilder retrieverBuilder, int rankWindowSize) { super(new ArrayList<>(), rankWindowSize); validateIdsAndDocs(ids, docs); - this.ids = ids != null ? ids : new ArrayList<>(); - this.docs = docs != null ? docs : new ArrayList<>(); + this.ids = ids; + this.docs = docs; addChild(new PinnedRetrieverBuilderWrapper(retrieverBuilder)); } @@ -180,10 +180,10 @@ protected SearchSourceBuilder finalizeSourceBuilder(SearchSourceBuilder source) @Override public void doToXContent(XContentBuilder builder, Params params) throws IOException { - if (ids != null && ids.isEmpty() == false) { + if (ids != null) { builder.array(IDS_FIELD.getPreferredName(), ids.toArray()); } - if (docs != null && docs.isEmpty() == false) { + if (docs != null) { builder.startArray(DOCS_FIELD.getPreferredName()); for (SpecifiedDocument doc : docs) { builder.value(doc); From 1c239d4a1c0b804de7a99458e44692e105a869ee Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 23 Apr 2025 02:29:43 +0000 Subject: [PATCH 52/83] [CI] Auto commit changes from spotless --- .../xpack/searchbusinessrules/retriever/PinnedRankDoc.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java index 6c1be44703efd..7c527a912132c 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.rank.RankDoc; -import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.Objects; From 38c5dbf7dd9d77f90ac647afba1713af019c3dc7 Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 23 Apr 2025 03:29:56 +0100 Subject: [PATCH 53/83] Retriever status changed to 9.1 version --- .../xpack/searchbusinessrules/retriever/PinnedRankDoc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java index 7c527a912132c..e49691d785096 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java @@ -76,6 +76,6 @@ protected int doHashCode() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.INITIAL_ELASTICSEARCH_9_0_1; + return TransportVersions.COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED; } } From 40cd06b4ff1721017d6f26eedd318d984e0bc80e Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 23 Apr 2025 03:43:04 +0100 Subject: [PATCH 54/83] cleaned up 2 yamltestsuite for different licenses --- ...BusinessRulesRrfClientYamlTestSuiteIT.java | 44 --------- .../10_pinned_retriever_basic.yml | 77 +++++++++++++++ .../11_pinned_retriever_rrf.yml | 97 ------------------- 3 files changed, 77 insertions(+), 141 deletions(-) delete mode 100644 x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesRrfClientYamlTestSuiteIT.java delete mode 100644 x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/11_pinned_retriever_rrf.yml diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesRrfClientYamlTestSuiteIT.java b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesRrfClientYamlTestSuiteIT.java deleted file mode 100644 index 2eb2ba645b6c3..0000000000000 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesRrfClientYamlTestSuiteIT.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.searchbusinessrules; - -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.cluster.local.distribution.DistributionType; -import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.junit.ClassRule; - -/** - * Test suite for running YAML tests specific to Pinned Retriever + RRF combination, - * requiring a trial license. - */ -public class SearchBusinessRulesRrfClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { - - @ClassRule - public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - // RRF requires trial or platinum license - .setting("xpack.license.self_generated.type", "trial") - .setting("xpack.security.enabled", "false") - .build(); - - public SearchBusinessRulesRrfClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { - super(testCandidate); - } - - @ParametersFactory - public static Iterable parameters() throws Exception { - return ESClientYamlSuiteTestCase.createParameters(new String[] { "search-business-rules/11_pinned_retriever_rrf" }); - } - - @Override - protected String getTestRestCluster() { - return cluster.getHttpAddresses(); - } -} diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever_basic.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever_basic.yml index 2e396c6fe7c60..ba5ad710f1f49 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever_basic.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever_basic.yml @@ -391,3 +391,80 @@ setup: ids: ["doc1"] retriever: standard: {} +--- +"pinned retriever combined with rrf": + - do: + cluster.put_settings: + body: + persistent: + xpack.license.self_generated.type: trial + - skip: + features: headers + + - do: + headers: + Content-Type: application/json + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1"] + retriever: + rrf: + retrievers: [ + { + standard: { + query: { + term: { text: "document" } + } + } + }, + { + standard: { + query: { + term: { text: "three" } + } + } + } + ] + rank_window_size: 10 + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014122E38 } + - match: { hits.hits.1._id: doc3 } + - match: { hits.hits.1._score < 100.0 } + - match: { hits.hits.2._id: doc2 } + +--- +"pinned retriever as a sub-retriever": + - do: + cluster.put_settings: + body: + persistent: + xpack.license.self_generated.type: trial + - skip: { features: headers } + - do: + headers: { Content-Type: application/json } + search: + index: test-index1 + body: + retriever: + rrf: + retrievers: + - + standard: + query: + match: { text: "document" } + - + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - lt: { hits.hits.0._score: 100.0 } diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/11_pinned_retriever_rrf.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/11_pinned_retriever_rrf.yml deleted file mode 100644 index fd7049b96c141..0000000000000 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/11_pinned_retriever_rrf.yml +++ /dev/null @@ -1,97 +0,0 @@ -setup: - - requires: - cluster_features: 'pinned_retriever_supported' - reason: 'test requires pinned retriever implementation' - - do: - indices.create: - index: test-index1 - - - do: - bulk: - refresh: true - index: test-index1 - body: - - index: - _id: doc1 - - { "text": "document one" } - - index: - _id: doc2 - - { "text": "document two" } - - index: - _id: doc3 - - { "text": "document three" } - - index: - _id: doc4 - - { "text": "document four" } - - index: - _id: doc5 - - { "text": "document five" } - ---- -"pinned retriever combined with rrf": - - skip: - features: headers - - - do: - headers: - Content-Type: application/json - search: - index: test-index1 - body: - retriever: - pinned: - ids: ["doc1"] - retriever: - rrf: - retrievers: [ - { - standard: { - query: { - term: { text: "document" } - } - } - }, - { - standard: { - query: { - term: { text: "three" } - } - } - } - ] - rank_window_size: 10 - - - match: { hits.total.value: 5 } - - match: { hits.hits.0._id: doc1 } - - match: { hits.hits.0._score: 1.7014122E38 } - - match: { hits.hits.1._id: doc3 } - - match: { hits.hits.1._score < 100.0 } - - match: { hits.hits.2._id: doc2 } - ---- -"pinned retriever as a sub-retriever": - - skip: { features: headers } - - do: - headers: { Content-Type: application/json } - search: - index: test-index1 - body: - retriever: - rrf: - retrievers: - - - standard: - query: - match: { text: "document" } - - - pinned: - ids: ["doc1", "doc2"] - retriever: - standard: - query: - match: { text: "document" } - - - match: { hits.total.value: 5 } - - match: { hits.hits.0._id: doc1 } - - lt: { hits.hits.0._score: 100.0 } - From 8d3a36b78222aaf06a9446f6ac791658118fdf72 Mon Sep 17 00:00:00 2001 From: Mridula Date: Thu, 24 Apr 2025 17:19:35 +0100 Subject: [PATCH 55/83] Trying to get the clustering works --- .../features/RetrieverFeatures.java | 19 +++++++++++++++++++ .../plugin/search-business-rules/build.gradle | 2 +- .../SearchBusinessRulesFeatures.java | 6 ++---- ...lasticsearch.features.FeatureSpecification | 1 + ...rchBusinessRulesClientYamlTestSuiteIT.java | 3 ++- ...ever_basic.yml => 10_pinned_retriever.yml} | 7 +++---- 6 files changed, 28 insertions(+), 10 deletions(-) create mode 100644 libs/features/src/main/java/org/elasticsearch/features/RetrieverFeatures.java create mode 100644 x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification rename x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/{10_pinned_retriever_basic.yml => 10_pinned_retriever.yml} (99%) diff --git a/libs/features/src/main/java/org/elasticsearch/features/RetrieverFeatures.java b/libs/features/src/main/java/org/elasticsearch/features/RetrieverFeatures.java new file mode 100644 index 0000000000000..b3e307ec12572 --- /dev/null +++ b/libs/features/src/main/java/org/elasticsearch/features/RetrieverFeatures.java @@ -0,0 +1,19 @@ +package org.elasticsearch.features; + +import org.elasticsearch.features.FeatureSpecification; +import org.elasticsearch.features.NodeFeature; + +import java.util.Set; + +/** + * Shared feature definitions for retrievers. + */ +public class RetrieverFeatures implements FeatureSpecification { + + public static final NodeFeature PINNED_RETRIEVER_FEATURE = new NodeFeature("pinned_retriever_supported"); + + @Override + public Set getFeatures() { + return Set.of(PINNED_RETRIEVER_FEATURE); + } +} \ No newline at end of file diff --git a/x-pack/plugin/search-business-rules/build.gradle b/x-pack/plugin/search-business-rules/build.gradle index ae591c0b30085..85bbfb748cf33 100644 --- a/x-pack/plugin/search-business-rules/build.gradle +++ b/x-pack/plugin/search-business-rules/build.gradle @@ -15,9 +15,9 @@ base { dependencies { compileOnly project(path: xpackModule('core')) compileOnly project(':server') - compileOnly project(':libs:x-content') testImplementation(testArtifact(project(xpackModule('core')))) testImplementation(testArtifact(project(':server'))) + clusterModules project(xpackModule('search-business-rules')) } tasks.named("yamlRestTest") { diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java index af3d8375a8c93..a9dcdac2b9952 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java @@ -12,17 +12,15 @@ import java.util.Set; -import static org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder.PINNED_RETRIEVER_FEATURE; - public class SearchBusinessRulesFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of(PINNED_RETRIEVER_FEATURE); + return Set.of(new NodeFeature("pinned_retriever_supported")); } @Override public Set getTestFeatures() { - return Set.of(); + return Set.of(new NodeFeature("pinned_retriever_supported")); } } diff --git a/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification new file mode 100644 index 0000000000000..4d8cab289f9a4 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -0,0 +1 @@ +org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRulesFeatures \ No newline at end of file diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java index 43b63ceed4193..62f743ebd6cb5 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java @@ -21,6 +21,7 @@ public class SearchBusinessRulesClientYamlTestSuiteIT extends ESClientYamlSuiteT .distribution(DistributionType.DEFAULT) .setting("xpack.license.self_generated.type", "basic") .setting("xpack.security.enabled", "false") + .module("search-business-rules") .build(); public SearchBusinessRulesClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { @@ -29,7 +30,7 @@ public SearchBusinessRulesClientYamlTestSuiteIT(final ClientYamlTestCandidate te @ParametersFactory public static Iterable parameters() throws Exception { - return ESClientYamlSuiteTestCase.createParameters(new String[] { "search-business-rules/10_pinned_retriever_basic" }); + return ESClientYamlSuiteTestCase.createParameters(new String[] { "search-business-rules/10_pinned_retriever" }); } @Override diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever_basic.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml similarity index 99% rename from x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever_basic.yml rename to x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index ba5ad710f1f49..49310fc2f61c8 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever_basic.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -391,16 +391,15 @@ setup: ids: ["doc1"] retriever: standard: {} + --- "pinned retriever combined with rrf": + - skip: { features: headers } - do: cluster.put_settings: body: persistent: xpack.license.self_generated.type: trial - - skip: - features: headers - - do: headers: Content-Type: application/json @@ -439,12 +438,12 @@ setup: --- "pinned retriever as a sub-retriever": + - skip: { features: headers } - do: cluster.put_settings: body: persistent: xpack.license.self_generated.type: trial - - skip: { features: headers } - do: headers: { Content-Type: application/json } search: From f63c78dba50ea2ee7194f62686c7123fca126902 Mon Sep 17 00:00:00 2001 From: Mridula Date: Thu, 24 Apr 2025 19:54:51 +0100 Subject: [PATCH 56/83] Cleaned up the yaml clustering and also reorganised the yaml tests --- x-pack/plugin/rank-rrf/build.gradle | 1 + .../test/rrf/950_pinned_interaction.yml | 96 +++++++++++++++++++ .../plugin/search-business-rules/build.gradle | 3 + .../retriever/PinnedRetrieverBuilder.java | 2 +- ...rchBusinessRulesClientYamlTestSuiteIT.java | 8 +- .../10_pinned_retriever.yml | 79 +-------------- 6 files changed, 108 insertions(+), 81 deletions(-) create mode 100644 x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/950_pinned_interaction.yml diff --git a/x-pack/plugin/rank-rrf/build.gradle b/x-pack/plugin/rank-rrf/build.gradle index 216e85f48f56f..fa598c6ef677a 100644 --- a/x-pack/plugin/rank-rrf/build.gradle +++ b/x-pack/plugin/rank-rrf/build.gradle @@ -26,6 +26,7 @@ dependencies { clusterModules project(xpackModule('rank-rrf')) clusterModules project(xpackModule('inference')) clusterModules project(':modules:lang-painless') + clusterModules project(xpackModule('search-business-rules')) clusterPlugins project(':x-pack:plugin:inference:qa:test-service-plugin') } diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/950_pinned_interaction.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/950_pinned_interaction.yml new file mode 100644 index 0000000000000..d3bffa8b297ed --- /dev/null +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/950_pinned_interaction.yml @@ -0,0 +1,96 @@ +setup: + - requires: + cluster_features: 'pinned_retriever_supported' + reason: 'test requires pinned retriever implementation' + - do: + indices.create: + index: test-index1 + + - do: + bulk: + refresh: true + index: test-index1 + body: + - index: + _id: doc1 + - { "text": "document one" } + - index: + _id: doc2 + - { "text": "document two" } + - index: + _id: doc3 + - { "text": "document three" } + - index: + _id: doc4 + - { "text": "document four" } + - index: + _id: doc5 + - { "text": "document five" } + +--- +"rrf combined with pinned retriever": + - skip: { features: headers } + - do: + headers: + Content-Type: application/json + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1"] + retriever: + rrf: + retrievers: [ + { + standard: { + query: { + term: { text: "document" } + } + } + }, + { + standard: { + query: { + term: { text: "three" } + } + } + } + ] + rank_window_size: 10 + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014122E38 } + - match: { hits.hits.1._id: doc3 } + - match: { hits.hits.1._score < 100.0 } + - match: { hits.hits.2._id: doc2 } + +--- +"rrf with pinned retriever as a sub-retriever": + - skip: { features: headers } + - do: + headers: { Content-Type: application/json } + search: + index: test-index1 + body: + retriever: + rrf: + retrievers: + - + standard: + query: + match: { text: "document" } + - + pinned: + ids: ["doc1", "doc2"] + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - lt: { hits.hits.0._score: 100.0 } + + diff --git a/x-pack/plugin/search-business-rules/build.gradle b/x-pack/plugin/search-business-rules/build.gradle index 85bbfb748cf33..98f79ee2f769b 100644 --- a/x-pack/plugin/search-business-rules/build.gradle +++ b/x-pack/plugin/search-business-rules/build.gradle @@ -18,6 +18,9 @@ dependencies { testImplementation(testArtifact(project(xpackModule('core')))) testImplementation(testArtifact(project(':server'))) clusterModules project(xpackModule('search-business-rules')) + clusterModules project(':modules:mapper-extras') + clusterModules project(':modules:lang-painless') + clusterModules project(xpackModule('inference')) } tasks.named("yamlRestTest") { diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index 04e83c8c25db4..c87436e9ea29f 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -165,7 +165,7 @@ public int rankWindowSize() { private QueryBuilder createPinnedQuery(QueryBuilder baseQuery) { Objects.requireNonNull(baseQuery, "Underlying query cannot be null for pinned retriever"); - if (docs.isEmpty() == false) { + if (docs != null && docs.isEmpty() == false) { return new PinnedQueryBuilder(baseQuery, docs.toArray(new SpecifiedDocument[0])); } return new PinnedQueryBuilder(baseQuery, ids.toArray(new String[0])); diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java index 62f743ebd6cb5..59971598269fa 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesClientYamlTestSuiteIT.java @@ -9,7 +9,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.junit.ClassRule; @@ -18,10 +17,13 @@ public class SearchBusinessRulesClientYamlTestSuiteIT extends ESClientYamlSuiteT @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) + .nodes(1) + .module("search-business-rules") + .module("mapper-extras") + .module("lang-painless") + .module("x-pack-inference") .setting("xpack.license.self_generated.type", "basic") .setting("xpack.security.enabled", "false") - .module("search-business-rules") .build(); public SearchBusinessRulesClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index 49310fc2f61c8..4961dc3c991f7 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -267,14 +267,14 @@ setup: - match: { hits.hits.0._id: doc1 } - gt: { hits.hits.0._score: 1.0 } - match: - hits.hits.0._explanation.description: "Pinned document by ids, original explanation:" + hits.hits.0._explanation.description: "Pinned document, original explanation:" - match: hits.hits.0._explanation.details.0.description: "doc [0] with an original score of [1.7014124E38] is at rank [1] from the following source queries." - match: { hits.hits.1._id: doc2 } - gt: { hits.hits.1._score: 1.0 } - match: - hits.hits.1._explanation.description: "Pinned document by ids, original explanation:" + hits.hits.1._explanation.description: "Pinned document, original explanation:" - match: hits.hits.1._explanation.details.0.description: "doc [1] with an original score of [1.7014122E38] is at rank [2] from the following source queries." @@ -392,78 +392,3 @@ setup: retriever: standard: {} ---- -"pinned retriever combined with rrf": - - skip: { features: headers } - - do: - cluster.put_settings: - body: - persistent: - xpack.license.self_generated.type: trial - - do: - headers: - Content-Type: application/json - search: - index: test-index1 - body: - retriever: - pinned: - ids: ["doc1"] - retriever: - rrf: - retrievers: [ - { - standard: { - query: { - term: { text: "document" } - } - } - }, - { - standard: { - query: { - term: { text: "three" } - } - } - } - ] - rank_window_size: 10 - - - match: { hits.total.value: 5 } - - match: { hits.hits.0._id: doc1 } - - match: { hits.hits.0._score: 1.7014122E38 } - - match: { hits.hits.1._id: doc3 } - - match: { hits.hits.1._score < 100.0 } - - match: { hits.hits.2._id: doc2 } - ---- -"pinned retriever as a sub-retriever": - - skip: { features: headers } - - do: - cluster.put_settings: - body: - persistent: - xpack.license.self_generated.type: trial - - do: - headers: { Content-Type: application/json } - search: - index: test-index1 - body: - retriever: - rrf: - retrievers: - - - standard: - query: - match: { text: "document" } - - - pinned: - ids: ["doc1", "doc2"] - retriever: - standard: - query: - match: { text: "document" } - - - match: { hits.total.value: 5 } - - match: { hits.hits.0._id: doc1 } - - lt: { hits.hits.0._score: 100.0 } From 2f1c37795e694baa4dcf2aba3aa3b92b9ce95ca8 Mon Sep 17 00:00:00 2001 From: Mridula Date: Thu, 24 Apr 2025 20:07:36 +0100 Subject: [PATCH 57/83] Unnecessary file introduction removed --- .../features/RetrieverFeatures.java | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 libs/features/src/main/java/org/elasticsearch/features/RetrieverFeatures.java diff --git a/libs/features/src/main/java/org/elasticsearch/features/RetrieverFeatures.java b/libs/features/src/main/java/org/elasticsearch/features/RetrieverFeatures.java deleted file mode 100644 index b3e307ec12572..0000000000000 --- a/libs/features/src/main/java/org/elasticsearch/features/RetrieverFeatures.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.elasticsearch.features; - -import org.elasticsearch.features.FeatureSpecification; -import org.elasticsearch.features.NodeFeature; - -import java.util.Set; - -/** - * Shared feature definitions for retrievers. - */ -public class RetrieverFeatures implements FeatureSpecification { - - public static final NodeFeature PINNED_RETRIEVER_FEATURE = new NodeFeature("pinned_retriever_supported"); - - @Override - public Set getFeatures() { - return Set.of(PINNED_RETRIEVER_FEATURE); - } -} \ No newline at end of file From c979859ac4a3f243f683fc75886db8c116acee0d Mon Sep 17 00:00:00 2001 From: Mridula Date: Thu, 24 Apr 2025 20:12:02 +0100 Subject: [PATCH 58/83] Reverted the plugins to the previous state as the changes werent necessary --- plugins/build.gradle | 44 ++++++++++++-------------------------------- 1 file changed, 12 insertions(+), 32 deletions(-) diff --git a/plugins/build.gradle b/plugins/build.gradle index f38525e46d51c..5c1b296c37fa8 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -11,37 +11,17 @@ configurations { allPlugins } -def pluginDirs = [ - rootProject.file('plugins'), - rootProject.file('x-pack/plugin') -] +// only configure immediate children of plugins dir +configure(subprojects.findAll { it.parent.path == project.path }) { + group = 'org.elasticsearch.plugin' + apply plugin: 'elasticsearch.internal-es-plugin' -pluginDirs.each { dir -> - if (dir.isDirectory()) { - dir.listFiles()?.each { subDir -> - if (subDir.isDirectory()) { - def proj = rootProject.findProject(":${dir.name}:${subDir.name}") - - if (proj != null) { - proj.configure(proj) { - group = 'org.elasticsearch.plugin' - apply plugin: 'elasticsearch.internal-es-plugin' - - esplugin { - name = proj.name - if (dir.name == 'x-pack') { - licenseFile = rootProject.layout.settingsDirectory.file('licenses/ELASTIC-LICENSE-2.0.txt').asFile - noticeFile = rootProject.file('x-pack/NOTICE.txt') - } else { - licenseFile = rootProject.layout.settingsDirectory.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile - noticeFile = rootProject.layout.settingsDirectory.file('NOTICE.txt').asFile - } - } - - rootProject.project(':plugins').artifacts.add('allPlugins', proj.tasks.named('bundlePlugin')) - } - } - } - } + esplugin { + // for local ES plugins, the name of the plugin is the same as the directory + name = project.name + licenseFile = layout.settingsDirectory.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile + noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile } -} + + parent.artifacts.add('allPlugins', tasks.named('bundlePlugin')) +} \ No newline at end of file From 527a9bfb2ceaff221cb18d80a276fa4d97107d8f Mon Sep 17 00:00:00 2001 From: Mridula Date: Thu, 24 Apr 2025 20:22:34 +0100 Subject: [PATCH 59/83] Removed unnecessary transport versioning from pinnedrankdoc --- .../xpack/searchbusinessrules/retriever/PinnedRankDoc.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java index e49691d785096..9ad682d5fb4b0 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java @@ -73,9 +73,4 @@ protected boolean doEquals(RankDoc rd) { protected int doHashCode() { return Objects.hash(super.doHashCode(), isPinned); } - - @Override - public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED; - } } From 42e5973139b58ce4c114f060becc90fd15e7e476 Mon Sep 17 00:00:00 2001 From: Mridula Date: Thu, 24 Apr 2025 20:23:21 +0100 Subject: [PATCH 60/83] reverted --- plugins/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/build.gradle b/plugins/build.gradle index 5c1b296c37fa8..90fc924d7f9d6 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -24,4 +24,4 @@ configure(subprojects.findAll { it.parent.path == project.path }) { } parent.artifacts.add('allPlugins', tasks.named('bundlePlugin')) -} \ No newline at end of file +} From f9ed13a36a1fe98f77981b6ce4ec0359d96d10fa Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 24 Apr 2025 19:30:02 +0000 Subject: [PATCH 61/83] [CI] Auto commit changes from spotless --- .../xpack/searchbusinessrules/retriever/PinnedRankDoc.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java index 9ad682d5fb4b0..04e5d29c3c36a 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java @@ -8,8 +8,6 @@ package org.elasticsearch.xpack.searchbusinessrules.retriever; import org.apache.lucene.search.Explanation; -import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.rank.RankDoc; From 1aa42f80ba40df4a0ffaac5dfcbbb9a56af0f040 Mon Sep 17 00:00:00 2001 From: Mridula Date: Fri, 25 Apr 2025 10:07:17 +0100 Subject: [PATCH 62/83] Edited the SearchBusinessRules to remove the class from getFeatures --- .../searchbusinessrules/SearchBusinessRulesFeatures.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java index a9dcdac2b9952..7767cba249543 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java @@ -10,17 +10,19 @@ import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; +import static org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder.PINNED_RETRIEVER_FEATURE; + import java.util.Set; public class SearchBusinessRulesFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of(new NodeFeature("pinned_retriever_supported")); + return Set.of(); } @Override public Set getTestFeatures() { - return Set.of(new NodeFeature("pinned_retriever_supported")); + return Set.of(PINNED_RETRIEVER_FEATURE); } } From 92993d0db8fa3a2021667a51952254875a5a3e18 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 25 Apr 2025 09:34:57 +0000 Subject: [PATCH 63/83] [CI] Auto commit changes from spotless --- .../searchbusinessrules/SearchBusinessRulesFeatures.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java index 7767cba249543..d702a0fcc6ea0 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java @@ -10,10 +10,10 @@ import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; -import static org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder.PINNED_RETRIEVER_FEATURE; - import java.util.Set; +import static org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder.PINNED_RETRIEVER_FEATURE; + public class SearchBusinessRulesFeatures implements FeatureSpecification { @Override From 018a525f4fb78c9d04bd98fe337e813a03ecd3ab Mon Sep 17 00:00:00 2001 From: Mridula Date: Fri, 25 Apr 2025 10:39:52 +0100 Subject: [PATCH 64/83] Cleaned up Pinned retriever to allow only id or docs --- .../retriever/PinnedRetrieverBuilder.java | 11 ++++++----- .../retriever/PinnedRetrieverBuilderTests.java | 8 ++++---- .../search-business-rules/10_pinned_retriever.yml | 7 ++----- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index c87436e9ea29f..2d07f291e8022 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -89,13 +89,14 @@ public static PinnedRetrieverBuilder fromXContent(XContentParser parser, Retriev private final List docs; private void validateIdsAndDocs(List ids, List docs) { - boolean hasIds = ids != null && ids.isEmpty() == false; - boolean hasDocs = docs != null && docs.isEmpty() == false; - - if (hasIds && hasDocs) { + if (ids != null && docs != null) { throw new IllegalArgumentException("Both 'ids' and 'docs' cannot be specified at the same time"); } - if (hasIds == false && hasDocs == false) { + + boolean validIds = ids != null && ids.isEmpty() == false; + boolean validDocs = docs != null && docs.isEmpty() == false; + + if (!validIds && !validDocs) { throw new IllegalArgumentException("Either 'ids' or 'docs' must be provided and non-empty for pinned retriever"); } } diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java index 63d5ef381deed..94b2b6dc4c147 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -57,9 +57,9 @@ public static PinnedRetrieverBuilder createRandomPinnedRetrieverBuilder() { List ids = useIds ? IntStream.range(0, numItems).mapToObj(i -> randomAlphaOfLengthBetween(5, 10)).collect(Collectors.toList()) - : new ArrayList<>(); + : null; List docs = useIds - ? new ArrayList<>() + ? null : IntStream.range(0, numItems) .mapToObj(i -> new SpecifiedDocument(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10))) .collect(Collectors.toList()); @@ -167,14 +167,14 @@ public void testValidation() { e = expectThrows(IllegalArgumentException.class, () -> { new PinnedRetrieverBuilder(List.of(), List.of(), new TestRetrieverBuilder("test"), DEFAULT_RANK_WINDOW_SIZE); }); - assertThat(e.getMessage(), equalTo("Either 'ids' or 'docs' must be provided and non-empty for pinned retriever")); + assertThat(e.getMessage(), equalTo("Both 'ids' and 'docs' cannot be specified at the same time")); } public void testValidateSort() { PinnedRetrieverBuilder builder = new PinnedRetrieverBuilder( List.of("id1"), - List.of(), + null, new TestRetrieverBuilder("test"), DEFAULT_RANK_WINDOW_SIZE ); diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index 4961dc3c991f7..f5f6277c46a96 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -345,8 +345,9 @@ setup: match: { text: "document" } --- -"pinned retriever with ids and empty docs array": +"pinned retriever with ids and empty docs array should fail": - do: + catch: /Both 'ids' and 'docs' cannot be specified at the same time/ search: index: test-index1 body: @@ -358,10 +359,6 @@ setup: standard: query: match: { text: "document" } - - match: { hits.total.value: 5 } - - match: { hits.hits.0._id: doc1 } - - match: { hits.hits.1._id: doc2 } - - match: { hits.hits.2._id: doc3 } --- "pinned retriever errors if sorting by anything other than score": From 137cc0dd2060d222b7514b193e3dff7b88ac7260 Mon Sep 17 00:00:00 2001 From: Mridula Date: Fri, 25 Apr 2025 11:00:47 +0100 Subject: [PATCH 65/83] Added more test to the pinned retriever --- .../10_pinned_retriever.yml | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index f5f6277c46a96..25d081555c4c8 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -389,3 +389,62 @@ setup: retriever: standard: {} +--- +"pinned retriever with non-existent id": + - skip: { features: headers } + - do: + headers: { Content-Type: application/json } + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "nonexistent_doc", "doc3"] + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - gt: { hits.hits.0._score: 1.0 } + - match: { hits.hits.1._id: doc3 } + - gt: { hits.hits.1._score: 1.0 } + - match: { hits.hits.2._id: doc2 } + - match: { hits.hits.2._score < 100.0 } + +--- +"pinned retriever with non-existent doc specified via docs": + - skip: { features: headers } + - do: + headers: { Content-Type: application/json } + search: + index: test-index1 + body: + retriever: + pinned: + docs: + - _index: test-index1 + _id: doc1 + - _index: test-index1 + _id: nonexistent_doc + - _index: test-index1 + _id: doc3 + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: doc1 } + - gt: { hits.hits.0._score: 1.0 } + - match: { hits.hits.1._id: doc3 } + - gt: { hits.hits.1._score: 1.0 } + - match: { hits.hits.2._id: doc2 } + - match: { hits.hits.2._score < 100.0 } + - match: { hits.hits.3._id: doc4 } + - match: { hits.hits.4._id: doc5 } + + + + From 152506978fd0bfa926108e7321f6d5230e123178 Mon Sep 17 00:00:00 2001 From: Mridula Date: Fri, 25 Apr 2025 11:01:40 +0100 Subject: [PATCH 66/83] did spotless --- .../retriever/PinnedRetrieverBuilderTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java index 94b2b6dc4c147..38a2299ca26f9 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilderTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.xpack.searchbusinessrules.SpecifiedDocument; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; From 74f7a95ec0a31edf34059653d632e76b5e55a5af Mon Sep 17 00:00:00 2001 From: Mridula Date: Fri, 25 Apr 2025 15:17:25 +0100 Subject: [PATCH 67/83] Introduced new transport versioning --- .../src/main/java/org/elasticsearch/TransportVersions.java | 2 +- .../xpack/searchbusinessrules/retriever/PinnedRankDoc.java | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index eaf3549bc83b1..f071b00dd7ed5 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -228,7 +228,7 @@ static TransportVersion def(int id) { public static final TransportVersion DENSE_VECTOR_OFF_HEAP_STATS = def(9_062_00_0); public static final TransportVersion RANDOM_SAMPLER_QUERY_BUILDER = def(9_063_0_00); public static final TransportVersion SETTINGS_IN_DATA_STREAMS = def(9_064_0_00); - + public static final TransportVersion PINNED_RETRIEVER = def(9_065_0_00); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java index 04e5d29c3c36a..ce2a3e217bb87 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java @@ -11,6 +11,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.rank.RankDoc; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; import java.io.IOException; import java.util.Objects; @@ -71,4 +73,9 @@ protected boolean doEquals(RankDoc rd) { protected int doHashCode() { return Objects.hash(super.doHashCode(), isPinned); } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.PINNED_RETRIEVER; + } } From dea8d6c0b8b172666c72887f9289f5867b5dfb72 Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 28 Apr 2025 10:28:25 +0100 Subject: [PATCH 68/83] Cleaning it up --- .../xpack/searchbusinessrules/retriever/PinnedRankDoc.java | 4 ++-- .../searchbusinessrules/retriever/PinnedRetrieverBuilder.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java index ce2a3e217bb87..fb13fad8720b1 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRankDoc.java @@ -8,11 +8,11 @@ package org.elasticsearch.xpack.searchbusinessrules.retriever; import org.apache.lucene.search.Explanation; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.rank.RankDoc; -import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; import java.io.IOException; import java.util.Objects; diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index 2d07f291e8022..3920665cf2ed1 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -96,7 +96,7 @@ private void validateIdsAndDocs(List ids, List docs) boolean validIds = ids != null && ids.isEmpty() == false; boolean validDocs = docs != null && docs.isEmpty() == false; - if (!validIds && !validDocs) { + if (validIds == false && validDocs == false) { throw new IllegalArgumentException("Either 'ids' or 'docs' must be provided and non-empty for pinned retriever"); } } From 5d6575a51630a7a05e0605013687aca26ce55512 Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 28 Apr 2025 16:25:33 +0100 Subject: [PATCH 69/83] Resolved validate module error --- .../plugin/search-business-rules/src/main/java/module-info.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/module-info.java b/x-pack/plugin/search-business-rules/src/main/java/module-info.java index fa736fe8c399b..aad2291cf82e5 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/module-info.java +++ b/x-pack/plugin/search-business-rules/src/main/java/module-info.java @@ -18,5 +18,5 @@ exports org.elasticsearch.xpack.searchbusinessrules; provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRulesFeatures; - + provides org.elasticsearch.plugins.SearchPlugin with org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRules; } From fae7a05c9aa8cbb16ea6397fc7d8fff951e35f11 Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 28 Apr 2025 20:29:44 +0100 Subject: [PATCH 70/83] BWT issues --- .../xpack/searchbusinessrules/SearchBusinessRules.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java index a6e990e69239b..468e0bf73a9d5 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -13,10 +13,13 @@ import org.elasticsearch.plugins.SearchPlugin.RetrieverSpec; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; import java.util.List; import static java.util.Collections.singletonList; +import static java.util.Collections.emptyList; public class SearchBusinessRules extends Plugin implements SearchPlugin { @@ -27,7 +30,10 @@ public List> getQueries() { @Override public List> getRetrievers() { - return singletonList(new RetrieverSpec<>(new ParseField(PinnedRetrieverBuilder.NAME), PinnedRetrieverBuilder::fromXContent)); + if (TransportVersion.current().onOrAfter(TransportVersions.PINNED_RETRIEVER)) { + return singletonList(new RetrieverSpec<>(new ParseField(PinnedRetrieverBuilder.NAME), PinnedRetrieverBuilder::fromXContent)); + } + return emptyList(); } } From 5b5ede486c4b5e664b0201b23adfb8aaed46bb30 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 28 Apr 2025 19:37:20 +0000 Subject: [PATCH 71/83] [CI] Auto commit changes from spotless --- .../xpack/searchbusinessrules/SearchBusinessRules.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java index 468e0bf73a9d5..7544076dc149e 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -7,19 +7,19 @@ package org.elasticsearch.xpack.searchbusinessrules; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.plugins.SearchPlugin.QuerySpec; import org.elasticsearch.plugins.SearchPlugin.RetrieverSpec; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder; -import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; import java.util.List; -import static java.util.Collections.singletonList; import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; public class SearchBusinessRules extends Plugin implements SearchPlugin { @@ -32,7 +32,7 @@ public List> getQueries() { public List> getRetrievers() { if (TransportVersion.current().onOrAfter(TransportVersions.PINNED_RETRIEVER)) { return singletonList(new RetrieverSpec<>(new ParseField(PinnedRetrieverBuilder.NAME), PinnedRetrieverBuilder::fromXContent)); - } + } return emptyList(); } From 215523dd456ca48a4ce9f928cf500ce40fbe7c2e Mon Sep 17 00:00:00 2001 From: Mridula Date: Mon, 28 Apr 2025 21:41:18 +0100 Subject: [PATCH 72/83] fix NPE occuring in ci --- .../retriever/PinnedRetrieverBuilder.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index 3920665cf2ed1..2d5addd7e6c5d 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -175,7 +175,11 @@ private QueryBuilder createPinnedQuery(QueryBuilder baseQuery) { @Override protected SearchSourceBuilder finalizeSourceBuilder(SearchSourceBuilder source) { validateSort(source); - source.query(createPinnedQuery(source.query())); + QueryBuilder underlyingQuery = source.query(); + if (underlyingQuery == null) { + throw new IllegalArgumentException("[underlying query] must not be null for pinned retriever"); + } + source.query(createPinnedQuery(underlyingQuery)); return source; } From 27a33a066d9512638ff34db9203028beb4a5baf0 Mon Sep 17 00:00:00 2001 From: Mridula Date: Tue, 29 Apr 2025 11:49:42 +0100 Subject: [PATCH 73/83] trying to fix duplicate feature issue --- .../xpack/searchbusinessrules/SearchBusinessRulesFeatures.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java index d702a0fcc6ea0..3922fc71d90c1 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java @@ -23,6 +23,6 @@ public Set getFeatures() { @Override public Set getTestFeatures() { - return Set.of(PINNED_RETRIEVER_FEATURE); + return Set.of(); } } From 4cfcbad1cd6d9172b26ec9ee55bf513f38f9bca7 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 29 Apr 2025 10:57:59 +0000 Subject: [PATCH 74/83] [CI] Auto commit changes from spotless --- .../xpack/searchbusinessrules/SearchBusinessRulesFeatures.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java index 3922fc71d90c1..613b5cd6d516f 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java @@ -12,8 +12,6 @@ import java.util.Set; -import static org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder.PINNED_RETRIEVER_FEATURE; - public class SearchBusinessRulesFeatures implements FeatureSpecification { @Override From 13f80fb59b65a66e98144537483d537c9b8f7142 Mon Sep 17 00:00:00 2001 From: Mridula Date: Tue, 29 Apr 2025 16:25:00 +0100 Subject: [PATCH 75/83] modified the tests --- .../xpack/searchbusinessrules/SearchBusinessRulesFeatures.java | 2 +- .../test/search-business-rules/10_pinned_retriever.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java index 613b5cd6d516f..8627440096235 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java @@ -21,6 +21,6 @@ public Set getFeatures() { @Override public Set getTestFeatures() { - return Set.of(); + return Set.of(PINNED_RETRIEVER_FEATURE); } } diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index 25d081555c4c8..33689083fac3c 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -379,7 +379,7 @@ setup: --- "pinned retriever error case - null inner query": - do: - catch: /Underlying query cannot be null for pinned retriever/ + catch: /illegal_argument_exception.+reason=\[underlying query\] must not be null for pinned retriever/ search: index: test-index1 body: From 247c78d6f65551d098e1ec7c0f0539615b0675e2 Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Wed, 30 Apr 2025 09:17:35 -0400 Subject: [PATCH 76/83] Playing with pinned retriever CI (#127530) * Fix compilation error * Remove SearchPlugin from META-INF * Remove duplicate FeatureSpecification * Add service as test resource * Move to test * More file moving * Remove from module-info, remove file * Make search business rules plugin extensible * fix ent search plugin * Delete dup files * [CI] Auto commit changes from spotless * Add module info back in --------- Co-authored-by: elasticsearchmachine --- x-pack/plugin/ent-search/build.gradle | 4 ++-- .../xpack/searchbusinessrules/SearchBusinessRules.java | 3 ++- .../searchbusinessrules/SearchBusinessRulesFeatures.java | 2 ++ .../services/org.elasticsearch.features.FeatureSpecification | 2 +- .../META-INF/services/org.elasticsearch.plugins.SearchPlugin | 1 - 5 files changed, 7 insertions(+), 5 deletions(-) delete mode 100644 x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.plugins.SearchPlugin diff --git a/x-pack/plugin/ent-search/build.gradle b/x-pack/plugin/ent-search/build.gradle index bca7547f0cd8c..b9802566b6c2f 100644 --- a/x-pack/plugin/ent-search/build.gradle +++ b/x-pack/plugin/ent-search/build.gradle @@ -8,7 +8,7 @@ esplugin { name = 'x-pack-ent-search' description = 'Elasticsearch Expanded Pack Plugin - Enterprise Search' classname = 'org.elasticsearch.xpack.application.EnterpriseSearch' - extendedPlugins = ['x-pack-core'] + extendedPlugins = ['x-pack-core', 'search-business-rules'] } base { @@ -17,7 +17,7 @@ base { dependencies { compileOnly project(path: xpackModule('core')) - implementation project(xpackModule('search-business-rules')) + compileOnly project(xpackModule('search-business-rules')) api project(':modules:lang-mustache') // JSON Schema dependencies diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java index 7544076dc149e..a0faa24634af7 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -9,6 +9,7 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; +import org.elasticsearch.plugins.ExtensiblePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.plugins.SearchPlugin.QuerySpec; @@ -21,7 +22,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; -public class SearchBusinessRules extends Plugin implements SearchPlugin { +public class SearchBusinessRules extends Plugin implements SearchPlugin, ExtensiblePlugin { @Override public List> getQueries() { diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java index 8627440096235..d702a0fcc6ea0 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRulesFeatures.java @@ -12,6 +12,8 @@ import java.util.Set; +import static org.elasticsearch.xpack.searchbusinessrules.retriever.PinnedRetrieverBuilder.PINNED_RETRIEVER_FEATURE; + public class SearchBusinessRulesFeatures implements FeatureSpecification { @Override diff --git a/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification index 4d8cab289f9a4..04e9c67d68039 100644 --- a/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification +++ b/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -1 +1 @@ -org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRulesFeatures \ No newline at end of file +org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRulesFeatures diff --git a/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.plugins.SearchPlugin b/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.plugins.SearchPlugin deleted file mode 100644 index d622876859864..0000000000000 --- a/x-pack/plugin/search-business-rules/src/main/resources/META-INF/services/org.elasticsearch.plugins.SearchPlugin +++ /dev/null @@ -1 +0,0 @@ -org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRules From cbb8ab1ee5214a0a30b28c18fb76b43008310e9b Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 30 Apr 2025 19:39:04 +0100 Subject: [PATCH 77/83] top document no pinned --- .../rest-api-spec/test/rrf/950_pinned_interaction.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/950_pinned_interaction.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/950_pinned_interaction.yml index d3bffa8b297ed..e5629b7715994 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/950_pinned_interaction.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/950_pinned_interaction.yml @@ -83,7 +83,7 @@ setup: match: { text: "document" } - pinned: - ids: ["doc1", "doc2"] + ids: ["doc4", "doc5"] retriever: standard: query: @@ -92,5 +92,7 @@ setup: - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } - lt: { hits.hits.0._score: 100.0 } + - match: { hits.hits.1._id: doc4 } + - match: { hits.hits.2._id: doc5 } From 9bfcb4b4afc991305ce672144935e8abbc3bdd5a Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 30 Apr 2025 19:42:30 +0100 Subject: [PATCH 78/83] Removing this to see if it works without this transport version check, as i had made changes related to the cluster error --- .../xpack/searchbusinessrules/SearchBusinessRules.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java index a0faa24634af7..c96f93d55a940 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -7,8 +7,6 @@ package org.elasticsearch.xpack.searchbusinessrules; -import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; import org.elasticsearch.plugins.ExtensiblePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; @@ -19,7 +17,6 @@ import java.util.List; -import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; public class SearchBusinessRules extends Plugin implements SearchPlugin, ExtensiblePlugin { @@ -31,10 +28,7 @@ public List> getQueries() { @Override public List> getRetrievers() { - if (TransportVersion.current().onOrAfter(TransportVersions.PINNED_RETRIEVER)) { - return singletonList(new RetrieverSpec<>(new ParseField(PinnedRetrieverBuilder.NAME), PinnedRetrieverBuilder::fromXContent)); - } - return emptyList(); + return singletonList(new RetrieverSpec<>(new ParseField(PinnedRetrieverBuilder.NAME), PinnedRetrieverBuilder::fromXContent)); } } From 227011b4b822d93ad5cf553076ac6095e4655be8 Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 30 Apr 2025 19:51:43 +0100 Subject: [PATCH 79/83] Fixed the retriever builder comments and error message --- .../retriever/PinnedRetrieverBuilder.java | 5 ++--- .../test/search-business-rules/10_pinned_retriever.yml | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java index 2d5addd7e6c5d..2e7c18e6a85b9 100644 --- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/retriever/PinnedRetrieverBuilder.java @@ -157,14 +157,13 @@ public int rankWindowSize() { /** * Creates a PinnedQueryBuilder with the appropriate pinned documents. - * Prioritizes docs over ids if both are present. * * @param baseQuery the base query to pin documents to * @return a PinnedQueryBuilder * @throws IllegalArgumentException if baseQuery is null */ private QueryBuilder createPinnedQuery(QueryBuilder baseQuery) { - Objects.requireNonNull(baseQuery, "Underlying query cannot be null for pinned retriever"); + Objects.requireNonNull(baseQuery, "pinned retriever requires retriever with associated query"); if (docs != null && docs.isEmpty() == false) { return new PinnedQueryBuilder(baseQuery, docs.toArray(new SpecifiedDocument[0])); @@ -177,7 +176,7 @@ protected SearchSourceBuilder finalizeSourceBuilder(SearchSourceBuilder source) validateSort(source); QueryBuilder underlyingQuery = source.query(); if (underlyingQuery == null) { - throw new IllegalArgumentException("[underlying query] must not be null for pinned retriever"); + throw new IllegalArgumentException("pinned retriever requires retriever with associated query"); } source.query(createPinnedQuery(underlyingQuery)); return source; diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index 33689083fac3c..5bf16e75f49e5 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -379,7 +379,7 @@ setup: --- "pinned retriever error case - null inner query": - do: - catch: /illegal_argument_exception.+reason=\[underlying query\] must not be null for pinned retriever/ + catch: /illegal_argument_exception.+reason=pinned retriever requires retriever with associated query/ search: index: test-index1 body: From 84608c01bef74f174d7f546e259b2f6c8d3a0aaa Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 30 Apr 2025 19:56:36 +0100 Subject: [PATCH 80/83] removed the unnencessary explain --- .../test/search-business-rules/10_pinned_retriever.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index 5bf16e75f49e5..847b6f62ea86f 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -112,7 +112,6 @@ setup: standard: query: match: { text: "document" } - explain: true - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } @@ -135,7 +134,6 @@ setup: standard: query: match: { text: "document" } - explain: true - match: { hits.total.value: 5 } - match: { hits.hits.0._id: doc1 } From 1a684420886d5417e81fc45e8c177aea3dfe402e Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 30 Apr 2025 20:16:03 +0100 Subject: [PATCH 81/83] Fixed all the tests in the yaml file --- .../10_pinned_retriever.yml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index 847b6f62ea86f..3e19c01846862 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -404,11 +404,11 @@ setup: match: { text: "document" } - match: { hits.total.value: 5 } - - match: { hits.hits.0._id: doc1 } - - gt: { hits.hits.0._score: 1.0 } - - match: { hits.hits.1._id: doc3 } - - gt: { hits.hits.1._score: 1.0 } - - match: { hits.hits.2._id: doc2 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014126E38 } + - match: { hits.hits.1._id: doc3 } + - match: { hits.hits.1._score: 1.7014122E38 } + - match: { hits.hits.2._id: doc2 } - match: { hits.hits.2._score < 100.0 } --- @@ -434,14 +434,14 @@ setup: match: { text: "document" } - match: { hits.total.value: 5 } - - match: { hits.hits.0._id: doc1 } - - gt: { hits.hits.0._score: 1.0 } - - match: { hits.hits.1._id: doc3 } - - gt: { hits.hits.1._score: 1.0 } - - match: { hits.hits.2._id: doc2 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014126E38 } + - match: { hits.hits.1._id: doc3 } + - match: { hits.hits.1._score: 1.7014122E38 } + - match: { hits.hits.2._id: doc2 } - match: { hits.hits.2._score < 100.0 } - - match: { hits.hits.3._id: doc4 } - - match: { hits.hits.4._id: doc5 } + - match: { hits.hits.3._id: doc4 } + - match: { hits.hits.4._id: doc5 } From 29d16a282022d9b0cf0d6aaa89a1efd4123c0a29 Mon Sep 17 00:00:00 2001 From: Mridula Date: Wed, 30 Apr 2025 20:52:19 +0100 Subject: [PATCH 82/83] added extra test case --- .../10_pinned_retriever.yml | 69 ++++++++++++++++++- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml index 3e19c01846862..d052e5ae847f3 100644 --- a/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml +++ b/x-pack/plugin/search-business-rules/src/yamlRestTest/resources/rest-api-spec/test/search-business-rules/10_pinned_retriever.yml @@ -5,6 +5,9 @@ setup: - do: indices.create: index: test-index1 + - do: + indices.create: + index: test-index2 - do: bulk: @@ -26,6 +29,17 @@ setup: - index: _id: doc5 - { "text": "document five" } + - do: + bulk: + refresh: true + index: test-index2 + body: + - index: + _id: idx2-docA + - { "text": "index two document A" } + - index: + _id: idx2-docB + - { "text": "index two document B" } --- "pinned retriever parameter variations": @@ -442,7 +456,56 @@ setup: - match: { hits.hits.2._score < 100.0 } - match: { hits.hits.3._id: doc4 } - match: { hits.hits.4._id: doc5 } - - - +--- +"pinned retriever multi-index vs single-index filtering": + - skip: { features: headers } + + - do: + headers: { Content-Type: application/json } + search: + index: test-index1,test-index2 + body: + retriever: + pinned: + ids: ["doc1", "idx2-docA"] + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 7 } + - length: { hits.hits: 7 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.1._id: idx2-docA } + - match: { hits.hits.2._id: idx2-docB } + - match: { hits.hits.3._id: doc2 } + - match: { hits.hits.4._id: doc3 } + - match: { hits.hits.5._id: doc4 } + - match: { hits.hits.6._id: doc5 } + + - do: + headers: { Content-Type: application/json } + search: + index: test-index1 + body: + retriever: + pinned: + ids: ["doc1", "idx2-docA"] + retriever: + standard: + query: + match: { text: "document" } + + - match: { hits.total.value: 5 } + - length: { hits.hits: 5 } + - match: { hits.hits.0._id: doc1 } + - match: { hits.hits.0._score: 1.7014124E38 } + - match: { hits.hits.1._id: doc2 } + - match: { hits.hits.2._id: doc3 } + - match: { hits.hits.3._id: doc4 } + - match: { hits.hits.4._id: doc5 } + + + + From 819e279bb6d8a8f7ab8d88ee51b63c79efaef7b7 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 30 Apr 2025 20:04:17 +0000 Subject: [PATCH 83/83] [CI] Auto commit changes from spotless --- server/src/main/java/org/elasticsearch/TransportVersions.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index cc35a66dac9f8..719137bd16989 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -231,7 +231,6 @@ static TransportVersion def(int id) { public static final TransportVersion INTRODUCE_FAILURES_LIFECYCLE = def(9_065_0_00); public static final TransportVersion PINNED_RETRIEVER = def(9_066_0_00); - /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _