diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java index 765c69e28a9ad..8be2317a9ee6f 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -148,16 +149,28 @@ public void chunkedInfer( } private RankedDocsResults makeResults(List input) { - List results = new ArrayList<>(); int totalResults = input.size(); - float minScore = random.nextFloat(-1f, 1f); - float resultDiff = 0.2f; - for (int i = 0; i < input.size(); i++) { - results.add( - new RankedDocsResults.RankedDoc(totalResults - 1 - i, minScore + resultDiff * (totalResults - i), input.get(i)) - ); + try { + List results = new ArrayList<>(); + for (int i = 0; i < totalResults; i++) { + results.add(new RankedDocsResults.RankedDoc(i, Float.parseFloat(input.get(i)), input.get(i))); + } + return new RankedDocsResults(results.stream().sorted(Comparator.reverseOrder()).toList()); + } catch (NumberFormatException ex) { + List results = new ArrayList<>(); + float minScore = random.nextFloat(-1f, 1f); + float resultDiff = 0.2f; + for (int i = 0; i < input.size(); i++) { + results.add( + new RankedDocsResults.RankedDoc( + totalResults - 1 - i, + minScore + resultDiff * (totalResults - i), + input.get(totalResults - 1 - i) + ) + ); + } + return new RankedDocsResults(results); } - return new RankedDocsResults(results); } protected ServiceSettings getServiceSettingsFromMap(Map serviceSettingsMap) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java index 8c16cbae82d1f..84ec9512a4d51 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java @@ -40,6 +40,9 @@ public Set getFeatures() { private static final NodeFeature SEMANTIC_TEXT_HIGHLIGHTER = new NodeFeature("semantic_text.highlighter"); private static final NodeFeature SEMANTIC_TEXT_HIGHLIGHTER_DEFAULT = new NodeFeature("semantic_text.highlighter.default"); private static final NodeFeature SEMANTIC_TEXT_MATCH_ALL_HIGHLIGHTER = new NodeFeature("semantic_text.match_all_highlighter"); + private static final NodeFeature TEST_RERANKING_SERVICE_PARSE_TEXT_AS_SCORE = new NodeFeature( + "test_reranking_service.parse_text_as_score" + ); @Override public Set getTestFeatures() { @@ -59,7 +62,8 @@ public Set getTestFeatures() { SemanticInferenceMetadataFieldsMapper.INFERENCE_METADATA_FIELDS_ENABLED_BY_DEFAULT, SEMANTIC_TEXT_HIGHLIGHTER_DEFAULT, SEMANTIC_KNN_FILTER_FIX, - SEMANTIC_TEXT_MATCH_ALL_HIGHLIGHTER + SEMANTIC_TEXT_MATCH_ALL_HIGHLIGHTER, + TEST_RERANKING_SERVICE_PARSE_TEXT_AS_SCORE ); } } diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml index a1c83d0d49d8a..dbbb6338ab1d0 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml @@ -2,7 +2,6 @@ setup: - skip: features: - close_to - - contains - requires: cluster_features: "text_similarity_reranker_retriever_supported" reason: semantic reranking introduced in 8.15.0 @@ -35,16 +34,8 @@ setup: type: keyword subtopic: type: keyword - - - do: - index: - index: test-index - id: doc_1 - body: - text: "As seen from Earth, a solar eclipse happens when the Moon is directly between the Earth and the Sun." - topic: [ "science" ] - subtopic: [ "technology" ] - refresh: true + inference_text_field: + type: text - do: index: @@ -54,6 +45,7 @@ setup: text: "The phases of the Moon come from the position of the Moon relative to the Earth and Sun." topic: [ "science" ] subtopic: [ "astronomy" ] + inference_text_field: "0" refresh: true - do: @@ -63,11 +55,27 @@ setup: body: text: "Sun Moon Lake is a lake in Nantou County, Taiwan. It is the largest lake in Taiwan." topic: [ "geography" ] + inference_text_field: "1" + refresh: true + + - do: + index: + index: test-index + id: doc_1 + body: + text: "As seen from Earth, a solar eclipse happens when the Moon is directly between the Earth and the Sun." + topic: [ "science" ] + subtopic: [ "technology" ] + inference_text_field: "-1" refresh: true --- "Simple text similarity rank retriever": + - requires: + cluster_features: "test_reranking_service.parse_text_as_score" + reason: test_reranking_service can now parse provided input as score to provide deterministic ranks + - do: search: index: test-index @@ -77,14 +85,37 @@ setup: retriever: text_similarity_reranker: retriever: + # this one returns docs 1 and 2 standard: query: - term: - topic: "science" + bool: { + should: [ + { + constant_score: { + filter: { + term: { + subtopic: "technology" + } + }, + boost: 10 + } + }, + { + constant_score: { + filter: { + term: { + subtopic: "astronomy" + } + }, + boost: 1 + } + } + ] + } rank_window_size: 10 inference_id: my-rerank-model inference_text: "How often does the moon hide the sun?" - field: text + field: inference_text_field size: 10 - match: { hits.total.value: 2 } @@ -96,6 +127,10 @@ setup: --- "Simple text similarity rank retriever and filtering": + - requires: + cluster_features: "test_reranking_service.parse_text_as_score" + reason: test_reranking_service can now parse provided input as score to provide deterministic ranks + - do: search: index: test-index @@ -105,6 +140,7 @@ setup: retriever: text_similarity_reranker: retriever: + # this one returns doc 1 standard: query: term: @@ -115,7 +151,7 @@ setup: rank_window_size: 10 inference_id: my-rerank-model inference_text: "How often does the moon hide the sun?" - field: text + field: inference_text_field size: 10 - match: { hits.total.value: 1 } @@ -145,7 +181,7 @@ setup: rank_window_size: 10 inference_id: i-dont-exist inference_text: "How often does the moon hide the sun?" - field: text + field: inference_text_field size: 10 --- @@ -171,13 +207,17 @@ setup: rank_window_size: 10 inference_id: i-dont-exist inference_text: "asdfasdf" - field: text + field: inference_text_field size: 10 --- "text similarity reranking with explain": + - requires: + cluster_features: "test_reranking_service.parse_text_as_score" + reason: test_reranking_service can now parse provided input as score to provide deterministic ranks + - do: search: index: test-index @@ -188,10 +228,32 @@ setup: text_similarity_reranker: { retriever: { + # this one returns doc 1 and 2 standard: { query: { - term: { - topic: "science" + bool: { + should: [ + { + constant_score: { + filter: { + term: { + subtopic: "technology" + } + }, + boost: 10 + } + }, + { + constant_score: { + filter: { + term: { + subtopic: "astronomy" + } + }, + boost: 1 + } + } + ] } } } @@ -199,17 +261,17 @@ setup: rank_window_size: 10, inference_id: my-rerank-model, inference_text: "How often does the moon hide the sun?", - field: text + field: inference_text_field } } size: 10 explain: true - - contains: { hits.hits: { _id: "doc_2" } } - - contains: { hits.hits: { _id: "doc_1" } } + - match: { hits.hits.0._id: "doc_2" } + - match: { hits.hits.1._id: "doc_1" } - - match: {hits.hits.0._explanation.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[text\\].*/" } - - match: {hits.hits.0._explanation.details.0.description: "/weight.*science.*/" } + - match: {hits.hits.0._explanation.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[inference_text_field\\].*/" } + - match: {hits.hits.0._explanation.details.0.details.0.description: "/subtopic.*astronomy.*/" } --- "text similarity reranker properly handles aliases": @@ -283,7 +345,7 @@ setup: rank_window_size: 10 inference_id: my-rerank-model inference_text: "How often does the moon hide the sun?" - field: text + field: inference_text_field size: 10 - match: { hits.total.value: 1 } diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml index d9db1fe387625..4ac0058b3cb24 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml @@ -41,15 +41,9 @@ setup: type: keyword integer: type: integer + inference_text_field: + type: text - - do: - index: - index: test-index - id: doc_1 - body: - text: "Sun Moon Lake is a lake in Nantou County, Taiwan. It is the largest lake in Taiwan." - topic: [ "geography" ] - integer: 1 - do: index: @@ -60,6 +54,7 @@ setup: topic: [ "science" ] subtopic: [ "astronomy" ] integer: 2 + inference_text_field: "0" - do: index: @@ -70,6 +65,17 @@ setup: topic: [ "science" ] subtopic: [ "technology" ] integer: 3 + inference_text_field: "1" + + - do: + index: + index: test-index + id: doc_1 + body: + text: "Sun Moon Lake is a lake in Nantou County, Taiwan. It is the largest lake in Taiwan." + topic: [ "geography" ] + integer: 1 + inference_text_field: "-1" - do: indices.refresh: {} @@ -77,6 +83,10 @@ setup: --- "rrf retriever with a nested text similarity reranker": + - requires: + cluster_features: "test_reranking_service.parse_text_as_score" + reason: test_reranking_service can now parse provided input as score to provide deterministic ranks + - do: search: index: test-index @@ -88,6 +98,7 @@ setup: retrievers: [ { + # this one retrieves docs 1 and 2 standard: { query: { bool: { @@ -122,13 +133,14 @@ setup: } }, { + # this one retrieves doc 2 text_similarity_reranker: { retriever: { standard: { query: { term: { - topic: "science" + subtopic: "astronomy" } } } @@ -136,7 +148,7 @@ setup: rank_window_size: 10, inference_id: my-rerank-model, inference_text: "How often does the moon hide the sun?", - field: text + field: inference_text_field } } ], @@ -151,20 +163,21 @@ setup: field: topic size: 10 - - match: { hits.total.value: 3 } - - length: { hits.hits: 2 } + - match: { hits.total.value: 2 } + - length: { hits.hits: 1 } - - contains: { hits.hits: { _id: "doc_1" } } - - contains: { hits.hits: { _id: "doc_3" } } + - match: { hits.hits.0._id: "doc_1" } - - match: { aggregations.topics.buckets.0.key: "science" } - - match: { aggregations.topics.buckets.0.doc_count: 2 } - - match: { aggregations.topics.buckets.1.key: "geography" } - - match: { aggregations.topics.buckets.1.doc_count: 1 } + - contains: { aggregations.topics.buckets: { key: "geography", doc_count: 1 } } + - contains: { aggregations.topics.buckets: { key: "science", doc_count: 1 } } --- "Text similarity reranker on top of an RRF retriever": + - requires: + cluster_features: "test_reranking_service.parse_text_as_score" + reason: test_reranking_service can now parse provided input as score to provide deterministic ranks + - do: search: index: test-index @@ -176,10 +189,12 @@ setup: text_similarity_reranker: { retriever: { + # this one retrieves docs 1 and 3 so final ranking should be 3 and 1 based on reranking service rrf: { retrievers: [ { + # this one retrieves docs 1 and 3 standard: { query: { bool: { @@ -214,6 +229,7 @@ setup: } }, { + # this one retrieves doc 1 standard: { query: { term: { @@ -230,7 +246,7 @@ setup: rank_window_size: 10, inference_id: my-rerank-model, inference_text: "How often does the moon hide the sun?", - field: text + field: inference_text_field } } size: 10 @@ -255,6 +271,10 @@ setup: --- "explain using rrf retriever and text-similarity": + - requires: + cluster_features: "test_reranking_service.parse_text_as_score" + reason: test_reranking_service can now parse provided input as score to provide deterministic ranks + - do: search: index: test-index @@ -266,6 +286,7 @@ setup: retrievers: [ { + # this one retrieves docs 1 and 2 standard: { query: { bool: { @@ -300,6 +321,7 @@ setup: } }, { + # this one retrieves doc 2 text_similarity_reranker: { retriever: { @@ -314,7 +336,7 @@ setup: rank_window_size: 10, inference_id: my-rerank-model, inference_text: "How often does the moon hide the sun?", - field: text + field: inference_text_field } } ], @@ -334,5 +356,5 @@ setup: - match: {hits.hits.0._explanation.details.0.details.0.details.0.description: "/ConstantScore.*/" } - match: {hits.hits.0._explanation.details.1.value: 1} - match: {hits.hits.0._explanation.details.1.description: "/rrf.score:.\\[0.5\\].*/" } - - match: {hits.hits.0._explanation.details.1.details.0.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[text\\].*/" } + - match: {hits.hits.0._explanation.details.1.details.0.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[inference_text_field\\].*/" } - match: {hits.hits.0._explanation.details.1.details.0.details.0.description: "/weight.*astronomy.*/" }