Skip to content

Commit 854a78e

Browse files
markjhoymarkjhoy
committed
complete unit tests; add yaml tests
1 parent 39406c3 commit 854a78e

File tree

5 files changed

+1541
-8
lines changed

5 files changed

+1541
-8
lines changed

‎server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapperTests.java‎

Lines changed: 220 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.elasticsearch.search.vectors.SparseVectorQueryWrapper;
4040
import org.elasticsearch.test.index.IndexVersionUtils;
4141
import org.elasticsearch.xcontent.XContentBuilder;
42+
import org.elasticsearch.xcontent.XContentParseException;
4243
import org.hamcrest.Matchers;
4344
import org.junit.AssumptionViolatedException;
4445

@@ -340,7 +341,7 @@ protected IndexVersion boostNotAllowedIndexVersion() {
340341
return NEW_SPARSE_VECTOR_INDEX_VERSION;
341342
}
342343

343-
public void testSparseVectorUnsupportedIndex() throws Exception {
344+
public void testSparseVectorUnsupportedIndex() {
344345
IndexVersion version = IndexVersionUtils.randomVersionBetween(
345346
random(),
346347
PREVIOUS_SPARSE_VECTOR_INDEX_VERSION,
@@ -352,6 +353,175 @@ public void testSparseVectorUnsupportedIndex() throws Exception {
352353
assertThat(e.getMessage(), containsString(SparseVectorFieldMapper.ERROR_MESSAGE_8X));
353354
}
354355

356+
public void testPruneMustBeBoolean() {
357+
Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> {
358+
b.field("type", "sparse_vector");
359+
b.startObject("index_options");
360+
b.field("prune", "othervalue");
361+
b.endObject();
362+
})));
363+
assertThat(e.getMessage(), containsString("[index_options] failed to parse field [prune]"));
364+
assertThat(e.getCause().getCause(), instanceOf(IllegalArgumentException.class));
365+
assertThat(e.getCause().getCause().getMessage(), containsString("Failed to parse value [othervalue] as only [true] or [false] are allowed."));
366+
}
367+
368+
public void testPruningConfigurationIsMap() {
369+
Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> {
370+
b.field("type", "sparse_vector");
371+
b.startObject("index_options");
372+
b.field("prune", true);
373+
b.field("pruning_config", "this_is_not_a_map");
374+
b.endObject();
375+
})));
376+
assertThat(e.getMessage(), containsString("[index_options] pruning_config doesn't support values of type:"));
377+
assertThat(e.getCause(), instanceOf(XContentParseException.class));
378+
assertThat(
379+
e.getCause().getMessage(),
380+
containsString("[index_options] pruning_config doesn't support values of type: VALUE_STRING")
381+
);
382+
}
383+
384+
public void testWithIndexOptionsPruningConfigPruneRequired() throws Exception {
385+
386+
Exception eTestPruneIsFalse = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> {
387+
b.field("type", "sparse_vector");
388+
b.startObject("index_options");
389+
b.field("prune", false);
390+
b.startObject("pruning_config");
391+
b.field("tokens_freq_ratio_threshold", 5.0);
392+
b.field("tokens_weight_threshold", 0.4);
393+
b.endObject();
394+
b.endObject();
395+
})));
396+
assertThat(eTestPruneIsFalse.getMessage(), containsString("[index_options] failed to parse field [pruning_config]"));
397+
assertThat(eTestPruneIsFalse.getCause().getCause().getCause(), instanceOf(IllegalArgumentException.class));
398+
assertThat(
399+
eTestPruneIsFalse.getCause().getCause().getCause().getMessage(),
400+
containsString("[index_options] field [pruning_config] should only be set if [prune] is set to true")
401+
);
402+
403+
Exception eTestPruneIsMissing = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> {
404+
b.field("type", "sparse_vector");
405+
b.startObject("index_options");
406+
b.startObject("pruning_config");
407+
b.field("tokens_freq_ratio_threshold", 5.0);
408+
b.field("tokens_weight_threshold", 0.4);
409+
b.endObject();
410+
b.endObject();
411+
})));
412+
assertThat(
413+
eTestPruneIsMissing.getMessage(),
414+
containsString("Failed to parse mapping: Failed to build [index_options] after last required field arrived")
415+
);
416+
assertThat(eTestPruneIsMissing.getCause().getCause(), instanceOf(IllegalArgumentException.class));
417+
assertThat(
418+
eTestPruneIsMissing.getCause().getCause().getMessage(),
419+
containsString("[index_options] field [pruning_config] should only be set if [prune] is set to true")
420+
);
421+
}
422+
423+
public void testTokensFreqRatioCorrect() {
424+
Exception eTestInteger = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> {
425+
b.field("type", "sparse_vector");
426+
b.startObject("index_options");
427+
b.field("prune", true);
428+
b.startObject("pruning_config");
429+
b.field("tokens_freq_ratio_threshold", "notaninteger");
430+
b.endObject();
431+
b.endObject();
432+
})));
433+
assertThat(
434+
eTestInteger.getMessage(),
435+
containsString("Failed to parse mapping: [0:0] [index_options] failed to parse field [pruning_config]")
436+
);
437+
assertThat(eTestInteger.getCause().getCause(), instanceOf(XContentParseException.class));
438+
assertThat(
439+
eTestInteger.getCause().getCause().getMessage(),
440+
containsString("[pruning_config] failed to parse field [tokens_freq_ratio_threshold]")
441+
);
442+
assertThat(eTestInteger.getCause().getCause().getCause(), instanceOf(NumberFormatException.class));
443+
assertThat(eTestInteger.getCause().getCause().getCause().getMessage(), containsString("For input string: \"notaninteger\""));
444+
445+
Exception eTestRangeLower = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> {
446+
b.field("type", "sparse_vector");
447+
b.startObject("index_options");
448+
b.field("prune", true);
449+
b.startObject("pruning_config");
450+
b.field("tokens_freq_ratio_threshold", -2);
451+
b.endObject();
452+
b.endObject();
453+
})));
454+
assertThat(eTestRangeLower.getMessage(), containsString("[index_options] failed to parse field [pruning_config]"));
455+
assertThat(eTestRangeLower.getCause().getCause().getCause(), instanceOf(IllegalArgumentException.class));
456+
assertThat(
457+
eTestRangeLower.getCause().getCause().getCause().getMessage(),
458+
containsString("[tokens_freq_ratio_threshold] must be between [1] and [100], got -2.0")
459+
);
460+
461+
Exception eTestRangeHigher = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> {
462+
b.field("type", "sparse_vector");
463+
b.startObject("index_options");
464+
b.field("prune", true);
465+
b.startObject("pruning_config");
466+
b.field("tokens_freq_ratio_threshold", 101);
467+
b.endObject();
468+
b.endObject();
469+
})));
470+
assertThat(eTestRangeHigher.getMessage(), containsString("[index_options] failed to parse field [pruning_config]"));
471+
assertThat(eTestRangeHigher.getCause().getCause().getCause(), instanceOf(IllegalArgumentException.class));
472+
assertThat(
473+
eTestRangeHigher.getCause().getCause().getCause().getMessage(),
474+
containsString("[tokens_freq_ratio_threshold] must be between [1] and [100], got 101.0")
475+
);
476+
}
477+
478+
public void testTokensWeightThresholdCorrect() {
479+
Exception eTestDouble = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> {
480+
b.field("type", "sparse_vector");
481+
b.startObject("index_options");
482+
b.field("prune", true);
483+
b.startObject("pruning_config");
484+
b.field("tokens_weight_threshold", "notadouble");
485+
b.endObject();
486+
b.endObject();
487+
})));
488+
assertThat(eTestDouble.getMessage(), containsString("[index_options] failed to parse field [pruning_config]"));
489+
assertThat(eTestDouble.getCause().getCause().getCause(), instanceOf(NumberFormatException.class));
490+
assertThat(eTestDouble.getCause().getCause().getCause().getMessage(), containsString("For input string: \"notadouble\""));
491+
492+
Exception eTestRangeLower = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> {
493+
b.field("type", "sparse_vector");
494+
b.startObject("index_options");
495+
b.field("prune", true);
496+
b.startObject("pruning_config");
497+
b.field("tokens_weight_threshold", -0.1);
498+
b.endObject();
499+
b.endObject();
500+
})));
501+
assertThat(eTestRangeLower.getMessage(), containsString("[index_options] failed to parse field [pruning_config]"));
502+
assertThat(eTestRangeLower.getCause().getCause().getCause(), instanceOf(IllegalArgumentException.class));
503+
assertThat(
504+
eTestRangeLower.getCause().getCause().getCause().getMessage(),
505+
containsString("[tokens_weight_threshold] must be between 0 and 1")
506+
);
507+
508+
Exception eTestRangeHigher = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> {
509+
b.field("type", "sparse_vector");
510+
b.startObject("index_options");
511+
b.field("prune", true);
512+
b.startObject("pruning_config");
513+
b.field("tokens_weight_threshold", 1.1);
514+
b.endObject();
515+
b.endObject();
516+
})));
517+
assertThat(eTestRangeHigher.getMessage(), containsString("[index_options] failed to parse field [pruning_config]"));
518+
assertThat(eTestRangeHigher.getCause().getCause().getCause(), instanceOf(IllegalArgumentException.class));
519+
assertThat(
520+
eTestRangeHigher.getCause().getCause().getCause().getMessage(),
521+
containsString("[tokens_weight_threshold] must be between 0 and 1")
522+
);
523+
}
524+
355525
private void withSearchExecutionContext(MapperService mapperService, CheckedConsumer<SearchExecutionContext, IOException> consumer)
356526
throws IOException {
357527
for (boolean store : new boolean[] { true, false }) {
@@ -374,13 +544,20 @@ private void withSearchExecutionContext(MapperService mapperService, CheckedCons
374544

375545
public void testTypeQueryFinalizationWithRandomOptions() throws Exception {
376546
for (int i = 0; i < 20; i++) {
377-
runTestTypeQueryFinalization(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
547+
runTestTypeQueryFinalization(
548+
randomBoolean(), // usePreviousIndex
549+
randomBoolean(), // useIndexOptionsDefaults
550+
randomBoolean(), // explicitIndexOptionsDoNotPrune
551+
randomBoolean(), // queryOverridesPruning
552+
randomBoolean() // queryOverrideExplicitFalse
553+
);
378554
}
379555
}
380556

381557
public void testTypeQueryFinalizationDefaultsCurrentVersion() throws Exception {
382-
IndexVersion version = IndexVersions.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT;
558+
IndexVersion version = IndexVersion.current();
383559
MapperService mapperService = createMapperService(version, fieldMapping(this::minimalMapping));
560+
384561
// query should be pruned by default on newer index versions
385562
performTypeQueryFinalizationTest(mapperService, null, null, null, true);
386563
}
@@ -392,13 +569,15 @@ public void testTypeQueryFinalizationDefaultsPreviousVersion() throws Exception
392569
IndexVersions.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT
393570
);
394571
MapperService mapperService = createMapperService(version, fieldMapping(this::minimalMapping));
572+
395573
// query should _not_ be pruned by default on older index versions
396574
performTypeQueryFinalizationTest(mapperService, null, null, null, false);
397575
}
398576

399577
public void testTypeQueryFinalizationWithIndexExplicit() throws Exception {
400-
IndexVersion version = IndexVersions.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT;
578+
IndexVersion version = IndexVersion.current();
401579
MapperService mapperService = createMapperService(version, fieldMapping(this::mappingWithIndexOptions));
580+
402581
// query should be pruned via explicit index options
403582
performTypeQueryFinalizationTest(
404583
mapperService,
@@ -410,10 +589,45 @@ public void testTypeQueryFinalizationWithIndexExplicit() throws Exception {
410589
}
411590

412591
public void testTypeQueryFinalizationWithIndexExplicitDoNotPrune() throws Exception {
413-
IndexVersion version = IndexVersions.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT;
592+
IndexVersion version = IndexVersion.current();
414593
MapperService mapperService = createMapperService(version, fieldMapping(this::mappingWithIndexOptionsPruneFalse));
594+
415595
// query should be pruned via explicit index options
416-
performTypeQueryFinalizationTest(mapperService, new SparseVectorFieldMapper.IndexOptions(false, null), null, null, false);
596+
performTypeQueryFinalizationTest(
597+
mapperService,
598+
new SparseVectorFieldMapper.IndexOptions(false, null),
599+
null,
600+
null,
601+
false
602+
);
603+
}
604+
605+
public void testTypeQueryFinalizationQueryOverridesPruning() throws Exception {
606+
IndexVersion version = IndexVersion.current();
607+
MapperService mapperService = createMapperService(version, fieldMapping(this::mappingWithIndexOptionsPruneFalse));
608+
609+
// query should still be pruned due to query builder setting it
610+
performTypeQueryFinalizationTest(
611+
mapperService,
612+
new SparseVectorFieldMapper.IndexOptions(false, null),
613+
true,
614+
new TokenPruningConfig(),
615+
true
616+
);
617+
}
618+
619+
public void testTypeQueryFinalizationQueryOverridesPruningOff() throws Exception {
620+
IndexVersion version = IndexVersion.current();
621+
MapperService mapperService = createMapperService(version, fieldMapping(this::mappingWithIndexOptionsPruneFalse));
622+
623+
// query should not pruned due to query builder setting it
624+
performTypeQueryFinalizationTest(
625+
mapperService,
626+
new SparseVectorFieldMapper.IndexOptions(true, new TokenPruningConfig()),
627+
false,
628+
null,
629+
false
630+
);
417631
}
418632

419633
private void performTypeQueryFinalizationTest(

‎x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ public void testSparseVector() throws Exception {
140140
tokens,
141141
null,
142142
null,
143-
null,
143+
false,
144144
null
145145
);
146146
NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder(fieldType.getChunksField().fullPath(), sparseQuery, ScoreMode.Max);
@@ -183,7 +183,7 @@ public void testNoSemanticField() throws Exception {
183183
tokens,
184184
null,
185185
null,
186-
null,
186+
false,
187187
null
188188
);
189189
var query = new BoolQueryBuilder().should(sparseQuery).should(new MatchAllQueryBuilder());

0 commit comments

Comments
 (0)