Skip to content

[9.0] Update test_reranking_service to try and parse provided inputs as scores (#122328) #129730

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 19, 2025
Merged

[9.0] Update test_reranking_service to try and parse provided inputs as scores (#122328) #129730

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 21 additions & 8 deletions ...n/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
Expand Down Expand Up @@ -148,16 +149,28 @@ public void chunkedInfer(
}

private RankedDocsResults makeResults(List<String> input) {
List<RankedDocsResults.RankedDoc> results = new ArrayList<>();
int totalResults = input.size();
float minScore = random.nextFloat(-1f, 1f);
float resultDiff = 0.2f;
for (int i = 0; i < input.size(); i++) {
results.add(
new RankedDocsResults.RankedDoc(totalResults - 1 - i, minScore + resultDiff * (totalResults - i), input.get(i))
);
try {
List<RankedDocsResults.RankedDoc> results = new ArrayList<>();
for (int i = 0; i < totalResults; i++) {
results.add(new RankedDocsResults.RankedDoc(i, Float.parseFloat(input.get(i)), input.get(i)));
}
return new RankedDocsResults(results.stream().sorted(Comparator.reverseOrder()).toList());
} catch (NumberFormatException ex) {
List<RankedDocsResults.RankedDoc> results = new ArrayList<>();
float minScore = random.nextFloat(-1f, 1f);
float resultDiff = 0.2f;
for (int i = 0; i < input.size(); i++) {
results.add(
new RankedDocsResults.RankedDoc(
totalResults - 1 - i,
minScore + resultDiff * (totalResults - i),
input.get(totalResults - 1 - i)
)
);
}
return new RankedDocsResults(results);
}
return new RankedDocsResults(results);
}

protected ServiceSettings getServiceSettingsFromMap(Map<String, Object> serviceSettingsMap) {
Expand Down
6 changes: 5 additions & 1 deletion ...k/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ public class InferenceFeatures implements FeatureSpecification {
private static final NodeFeature SEMANTIC_TEXT_HIGHLIGHTER = new NodeFeature("semantic_text.highlighter");
private static final NodeFeature SEMANTIC_TEXT_HIGHLIGHTER_DEFAULT = new NodeFeature("semantic_text.highlighter.default");
private static final NodeFeature SEMANTIC_TEXT_MATCH_ALL_HIGHLIGHTER = new NodeFeature("semantic_text.match_all_highlighter");
private static final NodeFeature TEST_RERANKING_SERVICE_PARSE_TEXT_AS_SCORE = new NodeFeature(
"test_reranking_service.parse_text_as_score"
);

@Override
public Set<NodeFeature> getTestFeatures() {
Expand All @@ -47,7 +50,8 @@ public Set<NodeFeature> getTestFeatures() {
SemanticInferenceMetadataFieldsMapper.INFERENCE_METADATA_FIELDS_ENABLED_BY_DEFAULT,
SEMANTIC_TEXT_HIGHLIGHTER_DEFAULT,
SEMANTIC_KNN_FILTER_FIX,
SEMANTIC_TEXT_MATCH_ALL_HIGHLIGHTER
SEMANTIC_TEXT_MATCH_ALL_HIGHLIGHTER,
TEST_RERANKING_SERVICE_PARSE_TEXT_AS_SCORE
);
}
}
112 changes: 87 additions & 25 deletions ...yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ setup:
- skip:
features:
- close_to
- contains
- requires:
test_runner_features: "close_to"

Expand Down Expand Up @@ -33,16 +32,8 @@ setup:
type: keyword
subtopic:
type: keyword

- do:
index:
index: test-index
id: doc_1
body:
text: "As seen from Earth, a solar eclipse happens when the Moon is directly between the Earth and the Sun."
topic: [ "science" ]
subtopic: [ "technology" ]
refresh: true
inference_text_field:
type: text

- do:
index:
Expand All @@ -52,6 +43,7 @@ setup:
text: "The phases of the Moon come from the position of the Moon relative to the Earth and Sun."
topic: [ "science" ]
subtopic: [ "astronomy" ]
inference_text_field: "0"
refresh: true

- do:
Expand All @@ -61,11 +53,27 @@ setup:
body:
text: "Sun Moon Lake is a lake in Nantou County, Taiwan. It is the largest lake in Taiwan."
topic: [ "geography" ]
inference_text_field: "1"
refresh: true

- do:
index:
index: test-index
id: doc_1
body:
text: "As seen from Earth, a solar eclipse happens when the Moon is directly between the Earth and the Sun."
topic: [ "science" ]
subtopic: [ "technology" ]
inference_text_field: "-1"
refresh: true

---
"Simple text similarity rank retriever":

- requires:
cluster_features: "test_reranking_service.parse_text_as_score"
reason: test_reranking_service can now parse provided input as score to provide deterministic ranks

- do:
search:
index: test-index
Expand All @@ -75,14 +83,37 @@ setup:
retriever:
text_similarity_reranker:
retriever:
# this one returns docs 1 and 2
standard:
query:
term:
topic: "science"
bool: {
should: [
{
constant_score: {
filter: {
term: {
subtopic: "technology"
}
},
boost: 10
}
},
{
constant_score: {
filter: {
term: {
subtopic: "astronomy"
}
},
boost: 1
}
}
]
}
rank_window_size: 10
inference_id: my-rerank-model
inference_text: "How often does the moon hide the sun?"
field: text
field: inference_text_field
size: 10

- match: { hits.total.value: 2 }
Expand All @@ -94,6 +125,10 @@ setup:
---
"Simple text similarity rank retriever and filtering":

- requires:
cluster_features: "test_reranking_service.parse_text_as_score"
reason: test_reranking_service can now parse provided input as score to provide deterministic ranks

- do:
search:
index: test-index
Expand All @@ -103,6 +138,7 @@ setup:
retriever:
text_similarity_reranker:
retriever:
# this one returns doc 1
standard:
query:
term:
Expand All @@ -113,7 +149,7 @@ setup:
rank_window_size: 10
inference_id: my-rerank-model
inference_text: "How often does the moon hide the sun?"
field: text
field: inference_text_field
size: 10

- match: { hits.total.value: 1 }
Expand Down Expand Up @@ -143,7 +179,7 @@ setup:
rank_window_size: 10
inference_id: i-dont-exist
inference_text: "How often does the moon hide the sun?"
field: text
field: inference_text_field
size: 10

---
Expand All @@ -169,13 +205,17 @@ setup:
rank_window_size: 10
inference_id: i-dont-exist
inference_text: "asdfasdf"
field: text
field: inference_text_field
size: 10


---
"text similarity reranking with explain":

- requires:
cluster_features: "test_reranking_service.parse_text_as_score"
reason: test_reranking_service can now parse provided input as score to provide deterministic ranks

- do:
search:
index: test-index
Expand All @@ -186,28 +226,50 @@ setup:
text_similarity_reranker: {
retriever:
{
# this one returns doc 1 and 2
standard: {
query: {
term: {
topic: "science"
bool: {
should: [
{
constant_score: {
filter: {
term: {
subtopic: "technology"
}
},
boost: 10
}
},
{
constant_score: {
filter: {
term: {
subtopic: "astronomy"
}
},
boost: 1
}
}
]
}
}
}
},
rank_window_size: 10,
inference_id: my-rerank-model,
inference_text: "How often does the moon hide the sun?",
field: text
field: inference_text_field
}
}
size: 10
explain: true

- contains: { hits.hits: { _id: "doc_2" } }
- contains: { hits.hits: { _id: "doc_1" } }
- match: { hits.hits.0._id: "doc_2" }
- match: { hits.hits.1._id: "doc_1" }

- match: {hits.hits.0._explanation.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[text\\].*/" }
- match: {hits.hits.0._explanation.details.0.description: "/weight.*science.*/" }
- match: {hits.hits.0._explanation.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[inference_text_field\\].*/" }
- match: {hits.hits.0._explanation.details.0.details.0.description: "/subtopic.*astronomy.*/" }

---
"text similarity reranker properly handles aliases":
Expand Down Expand Up @@ -281,7 +343,7 @@ setup:
rank_window_size: 10
inference_id: my-rerank-model
inference_text: "How often does the moon hide the sun?"
field: text
field: inference_text_field
size: 10

- match: { hits.total.value: 1 }
Expand Down
Loading