Skip to content

Commit a1f177b

Browse files
markjhoymarkjhoy
committed
clean tests; add YAML Rest tests
1 parent cd26f2d commit a1f177b

File tree

6 files changed

+731
-25
lines changed

6 files changed

+731
-25
lines changed

‎x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_EXCLUDE_SUB_FIELDS_FROM_FIELD_CAPS;
2020
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_INDEX_OPTIONS;
2121
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_INDEX_OPTIONS_WITH_DEFAULTS;
22+
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_SPARSE_VECTOR_INDEX_OPTIONS;
2223
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_SUPPORT_CHUNKING_CONFIG;
2324
import static org.elasticsearch.xpack.inference.queries.SemanticKnnVectorQueryRewriteInterceptor.SEMANTIC_KNN_FILTER_FIX;
2425
import static org.elasticsearch.xpack.inference.queries.SemanticKnnVectorQueryRewriteInterceptor.SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED;
@@ -78,7 +79,8 @@ public Set<NodeFeature> getTestFeatures() {
7879
COHERE_V2_API,
7980
SEMANTIC_TEXT_INDEX_OPTIONS_WITH_DEFAULTS,
8081
SEMANTIC_QUERY_REWRITE_INTERCEPTORS_PROPAGATE_BOOST_AND_QUERY_NAME_FIX,
81-
SEMANTIC_TEXT_HIGHLIGHTING_FLAT
82+
SEMANTIC_TEXT_HIGHLIGHTING_FLAT,
83+
SEMANTIC_TEXT_SPARSE_VECTOR_INDEX_OPTIONS
8284
)
8385
);
8486
if (RERANK_SNIPPETS.isEnabled()) {

‎x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,8 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
143143
public static final NodeFeature SEMANTIC_TEXT_INDEX_OPTIONS_WITH_DEFAULTS = new NodeFeature(
144144
"semantic_text.index_options_with_defaults"
145145
);
146+
public static final NodeFeature SEMANTIC_TEXT_SPARSE_VECTOR_INDEX_OPTIONS =
147+
new NodeFeature("semantic_text.sparse_vector_index_options");
146148

147149
public static final String CONTENT_TYPE = "semantic_text";
148150
public static final String DEFAULT_ELSER_2_INFERENCE_ID = DEFAULT_ELSER_ID;

‎x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java‎

Lines changed: 41 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -722,18 +722,26 @@ public void testSparseVectorMappingUpdate() throws IOException {
722722
String fieldName = "field";
723723

724724
MapperService mapperService = createMapperService(
725-
mapping(b -> addSemanticTextMapping(b, fieldName, model.getInferenceEntityId(), null, chunkingSettings, indexOptions)),
725+
mapping(b -> addSemanticTextMapping(
726+
b,
727+
fieldName,
728+
model.getInferenceEntityId(),
729+
null,
730+
chunkingSettings,
731+
indexOptions,
732+
new MinimalServiceSettings(model)
733+
)),
726734
useLegacyFormat
727735
);
728-
assertSemanticTextField(mapperService, fieldName, false, chunkingSettings, indexOptions);
736+
assertSemanticTextField(mapperService, fieldName, true, chunkingSettings, indexOptions);
729737

730738
final SemanticTextIndexOptions newIndexOptions = randomSemanticTextIndexOptions(TaskType.SPARSE_EMBEDDING);
731739
ChunkingSettings newChunkingSettings = generateRandomChunkingSettingsOtherThan(chunkingSettings);
732740
merge(
733741
mapperService,
734742
mapping(b -> addSemanticTextMapping(b, fieldName, model.getInferenceEntityId(), null, newChunkingSettings, newIndexOptions))
735743
);
736-
assertSemanticTextField(mapperService, fieldName, false, newChunkingSettings, newIndexOptions);
744+
assertSemanticTextField(mapperService, fieldName, true, newChunkingSettings, newIndexOptions);
737745
}
738746
}
739747

@@ -926,27 +934,29 @@ public void testSuccessfulParse() throws IOException {
926934
model1.getInferenceEntityId(),
927935
setSearchInferenceId ? searchInferenceId : null,
928936
chunkingSettings,
929-
indexOptions
937+
indexOptions,
938+
new MinimalServiceSettings(model1)
930939
);
931940
addSemanticTextMapping(
932941
b,
933942
fieldName2,
934943
model2.getInferenceEntityId(),
935944
setSearchInferenceId ? searchInferenceId : null,
936945
chunkingSettings,
937-
indexOptions
946+
indexOptions,
947+
new MinimalServiceSettings(model2)
938948
);
939949
});
940950

941951
MapperService mapperService = createMapperService(mapping, useLegacyFormat);
942-
assertSemanticTextField(mapperService, fieldName1, false, null, indexOptions);
952+
assertSemanticTextField(mapperService, fieldName1, true, null, indexOptions);
943953
assertInferenceEndpoints(
944954
mapperService,
945955
fieldName1,
946956
model1.getInferenceEntityId(),
947957
setSearchInferenceId ? searchInferenceId : model1.getInferenceEntityId()
948958
);
949-
assertSemanticTextField(mapperService, fieldName2, false, null, indexOptions);
959+
assertSemanticTextField(mapperService, fieldName2, true, null, indexOptions);
950960
assertInferenceEndpoints(
951961
mapperService,
952962
fieldName2,
@@ -1207,7 +1217,14 @@ public void testModelSettingsRequiredWithChunks() throws IOException {
12071217
);
12081218

12091219
MapperService mapperService = createMapperService(
1210-
mapping(b -> addSemanticTextMapping(b, "field", model.getInferenceEntityId(), null, chunkingSettings, indexOptions)),
1220+
mapping(b -> addSemanticTextMapping(
1221+
b,
1222+
"field",
1223+
model.getInferenceEntityId(),
1224+
null,
1225+
chunkingSettings,
1226+
indexOptions
1227+
)),
12111228
useLegacyFormat
12121229
);
12131230
SourceToParse source = source(b -> addSemanticTextInferenceResults(useLegacyFormat, b, List.of(inferenceResults)));
@@ -1709,18 +1726,33 @@ protected void assertExistsQuery(MappedFieldType fieldType, Query query, LuceneD
17091726
// Until a doc is indexed, the query is rewritten as match no docs
17101727
assertThat(query, instanceOf(MatchNoDocsQuery.class));
17111728
}
1712-
17131729
private static void addSemanticTextMapping(
17141730
XContentBuilder mappingBuilder,
17151731
String fieldName,
17161732
String inferenceId,
17171733
String searchInferenceId,
17181734
ChunkingSettings chunkingSettings,
17191735
SemanticTextIndexOptions indexOptions
1736+
) throws IOException {
1737+
addSemanticTextMapping(mappingBuilder, fieldName, inferenceId, searchInferenceId, chunkingSettings, indexOptions, null);
1738+
}
1739+
1740+
private static void addSemanticTextMapping(
1741+
XContentBuilder mappingBuilder,
1742+
String fieldName,
1743+
String inferenceId,
1744+
String searchInferenceId,
1745+
ChunkingSettings chunkingSettings,
1746+
SemanticTextIndexOptions indexOptions,
1747+
MinimalServiceSettings modelSettings
17201748
) throws IOException {
17211749
mappingBuilder.startObject(fieldName);
17221750
mappingBuilder.field("type", SemanticTextFieldMapper.CONTENT_TYPE);
17231751
mappingBuilder.field("inference_id", inferenceId);
1752+
if (modelSettings != null) {
1753+
mappingBuilder.field("model_settings");
1754+
modelSettings.toXContent(mappingBuilder, null);
1755+
}
17241756
if (searchInferenceId != null) {
17251757
mappingBuilder.field("search_inference_id", searchInferenceId);
17261758
}

‎x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java‎

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -52,13 +52,10 @@ public static TestModel createRandomInstance(TaskType taskType, List<SimilarityM
5252
}
5353

5454
public static TestModel createRandomInstance(TaskType taskType, List<SimilarityMeasure> excludedSimilarities, int maxDimensions) {
55-
var elementType = taskType == TaskType.TEXT_EMBEDDING ? randomFrom(DenseVectorFieldMapper.ElementType.values()) : null;
56-
var dimensions = taskType == TaskType.TEXT_EMBEDDING
57-
? DenseVectorFieldMapperTestUtils.randomCompatibleDimensions(elementType, maxDimensions)
58-
: null;
59-
60-
SimilarityMeasure similarity = null;
6155
if (taskType == TaskType.TEXT_EMBEDDING) {
56+
var elementType = randomFrom(DenseVectorFieldMapper.ElementType.values());
57+
var dimensions = DenseVectorFieldMapperTestUtils.randomCompatibleDimensions(elementType, maxDimensions);
58+
6259
List<SimilarityMeasure> supportedSimilarities = new ArrayList<>(
6360
DenseVectorFieldMapperTestUtils.getSupportedSimilarities(elementType)
6461
);
@@ -75,17 +72,30 @@ public static TestModel createRandomInstance(TaskType taskType, List<SimilarityM
7572
);
7673
}
7774

78-
similarity = randomFrom(supportedSimilarities);
75+
SimilarityMeasure similarity = randomFrom(supportedSimilarities);
76+
77+
return new TestModel(
78+
randomAlphaOfLength(4),
79+
TaskType.TEXT_EMBEDDING,
80+
randomAlphaOfLength(10),
81+
new TestModel.TestServiceSettings(randomAlphaOfLength(4), dimensions, similarity, elementType),
82+
new TestModel.TestTaskSettings(randomInt(3)),
83+
new TestModel.TestSecretSettings(randomAlphaOfLength(4))
84+
);
85+
}
86+
87+
if (taskType == TaskType.SPARSE_EMBEDDING) {
88+
return new TestModel(
89+
randomAlphaOfLength(4),
90+
TaskType.SPARSE_EMBEDDING,
91+
randomAlphaOfLength(10),
92+
new TestModel.TestServiceSettings(randomAlphaOfLength(4), null, null, null),
93+
new TestModel.TestTaskSettings(randomInt(3)),
94+
new TestModel.TestSecretSettings(randomAlphaOfLength(4))
95+
);
7996
}
8097

81-
return new TestModel(
82-
randomAlphaOfLength(4),
83-
taskType,
84-
randomAlphaOfLength(10),
85-
new TestModel.TestServiceSettings(randomAlphaOfLength(4), dimensions, similarity, elementType),
86-
new TestModel.TestTaskSettings(randomInt(3)),
87-
new TestModel.TestSecretSettings(randomAlphaOfLength(4))
88-
);
98+
throw new IllegalArgumentException("Unsupported task type [" + taskType + "]");
8999
}
90100

91101
public TestModel(

0 commit comments

Comments
 (0)