Skip to content

Commit 3f67340

Browse files
mromaiosmromaios
committed
improve index testing coverage
1 parent d9cdbc8 commit 3f67340

File tree

1 file changed

+49
-21
lines changed

1 file changed

+49
-21
lines changed

‎server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapperTests.java‎

Lines changed: 49 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.elasticsearch.inference.WeightedToken;
4040
import org.elasticsearch.search.lookup.Source;
4141
import org.elasticsearch.search.vectors.SparseVectorQueryWrapper;
42+
import org.elasticsearch.test.ESTestCase;
4243
import org.elasticsearch.test.index.IndexVersionUtils;
4344
import org.elasticsearch.xcontent.ToXContent;
4445
import org.elasticsearch.xcontent.XContentBuilder;
@@ -54,16 +55,17 @@
5455
import java.util.LinkedHashMap;
5556
import java.util.List;
5657
import java.util.Map;
58+
import java.util.NavigableSet;
5759
import java.util.Set;
5860
import java.util.TreeMap;
5961
import java.util.stream.Collectors;
6062
import java.util.stream.Stream;
6163

64+
import static org.elasticsearch.index.IndexVersions.NEW_SPARSE_VECTOR;
6265
import static org.elasticsearch.index.IndexVersions.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT;
66+
import static org.elasticsearch.index.IndexVersions.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X;
6367
import static org.elasticsearch.index.IndexVersions.UPGRADE_TO_LUCENE_10_0_0;
64-
import static org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper.NEW_SPARSE_VECTOR_INDEX_VERSION;
65-
import static org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper.PREVIOUS_SPARSE_VECTOR_INDEX_VERSION;
66-
import static org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_VERSION;
68+
import static org.elasticsearch.index.IndexVersions.V_8_0_0;
6769
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
6870
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
6971
import static org.hamcrest.Matchers.containsString;
@@ -290,7 +292,7 @@ public void testDefaultsWithAndWithoutIncludeDefaultsOlderIndexVersion() throws
290292
IndexVersion indexVersion = IndexVersionUtils.randomVersionBetween(
291293
random(),
292294
UPGRADE_TO_LUCENE_10_0_0,
293-
IndexVersionUtils.getPreviousVersion(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_VERSION)
295+
IndexVersionUtils.getPreviousVersion(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X)
294296
);
295297

296298
XContentBuilder orig = JsonXContent.contentBuilder().startObject();
@@ -511,21 +513,21 @@ protected IngestScriptSupport ingestScriptSupport() {
511513
@Override
512514
protected String[] getParseMinimalWarnings(IndexVersion indexVersion) {
513515
String[] additionalWarnings = null;
514-
if (indexVersion.before(PREVIOUS_SPARSE_VECTOR_INDEX_VERSION)) {
516+
if (indexVersion.before(V_8_0_0)) {
515517
additionalWarnings = new String[] { SparseVectorFieldMapper.ERROR_MESSAGE_7X };
516518
}
517519
return Strings.concatStringArrays(super.getParseMinimalWarnings(indexVersion), additionalWarnings);
518520
}
519521

520522
@Override
521523
protected IndexVersion boostNotAllowedIndexVersion() {
522-
return NEW_SPARSE_VECTOR_INDEX_VERSION;
524+
return NEW_SPARSE_VECTOR;
523525
}
524526

525527
public void testSparseVectorUnsupportedIndex() {
526528
IndexVersion version = IndexVersionUtils.randomVersionBetween(
527529
random(),
528-
PREVIOUS_SPARSE_VECTOR_INDEX_VERSION,
530+
V_8_0_0,
529531
IndexVersions.FIRST_DETACHED_INDEX_VERSION
530532
);
531533
Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(version, fieldMapping(b -> {
@@ -752,9 +754,11 @@ private void withSearchExecutionContext(MapperService mapperService, CheckedCons
752754

753755
iw.addDocument(mapper.parse(source(b -> b.field("field", RARE_TOKENS))).rootDoc());
754756

755-
// This will lower the averageTokenFreqRatio so that common tokens get pruned with default settings
757+
// This will lower the averageTokenFreqRatio so that common tokens get pruned with default settings.
758+
// Depending on how the index is created, we will have 30-37 numUniqueTokens
759+
// this will result in an averageTokenFreqRatio of 0.1021 - 0.1259
756760
Map<String, Float> uniqueDoc = new TreeMap<>();
757-
for (int i = 0; i < 20; i++) {
761+
for (int i = 0; i < 30; i++) {
758762
uniqueDoc.put("unique" + i, 0.5f);
759763
}
760764
iw.addDocument(mapper.parse(source(b -> b.field("field", uniqueDoc))).rootDoc());
@@ -768,10 +772,10 @@ private void withSearchExecutionContext(MapperService mapperService, CheckedCons
768772
}
769773

770774
public void testPruningScenarios() throws Exception {
771-
for (int i = 0; i < 120; i++) {
775+
for (int i = 0; i < 200; i++) {
772776
assertPruningScenario(
773777
randomFrom(validIndexPruningScenarios),
774-
new PruningOptions(randomBoolean() ? randomBoolean() : null, randomFrom(PruningConfig.values()))
778+
new PruningOptions(randomFrom(true, false, null), randomFrom(PruningConfig.values()))
775779
);
776780
}
777781
}
@@ -801,7 +805,8 @@ private PruningScenario getEffectivePruningScenario(
801805
}
802806

803807
if (shouldPrune == null) {
804-
shouldPrune = indexVersion.onOrAfter(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT);
808+
shouldPrune = indexVersion.between(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X, UPGRADE_TO_LUCENE_10_0_0)
809+
|| indexVersion.onOrAfter(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT);
805810
}
806811

807812
PruningScenario pruningScenario = PruningScenario.NO_PRUNING;
@@ -839,7 +844,7 @@ private List<Query> getExpectedQueryClauses(
839844
}
840845

841846
private void assertPruningScenario(PruningOptions indexPruningOptions, PruningOptions queryPruningOptions) throws IOException {
842-
IndexVersion indexVersion = getIndexVersionForTest(randomBoolean());
847+
IndexVersion indexVersion = getIndexVersion();
843848
MapperService mapperService = createMapperService(indexVersion, getIndexMapping(indexPruningOptions));
844849
PruningScenario effectivePruningScenario = getEffectivePruningScenario(indexPruningOptions, queryPruningOptions, indexVersion);
845850
withSearchExecutionContext(mapperService, (context) -> {
@@ -858,16 +863,39 @@ private void assertPruningScenario(PruningOptions indexPruningOptions, PruningOp
858863
});
859864
}
860865

861-
private IndexVersion getIndexVersionForTest(boolean usePreviousIndex) {
862-
return usePreviousIndex
863-
? IndexVersionUtils.randomVersionBetween(
864-
random(),
865-
UPGRADE_TO_LUCENE_10_0_0,
866-
IndexVersionUtils.getPreviousVersion(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT)
867-
)
868-
: IndexVersionUtils.randomVersionBetween(random(), SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT, IndexVersion.current());
866+
private static IndexVersion getIndexVersion() {
867+
VersionRange versionRange = randomFrom(VersionRange.values());
868+
return versionRange.getRandomVersion();
869869
}
870870

871+
private enum VersionRange {
872+
ES_V8X_WITHOUT_SUPPORT(
873+
NEW_SPARSE_VECTOR,
874+
IndexVersionUtils.getPreviousVersion(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X)
875+
),
876+
ES_V8X_WITH_SUPPORT(
877+
SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X,
878+
IndexVersionUtils.getPreviousVersion(UPGRADE_TO_LUCENE_10_0_0)
879+
),
880+
ES_V9X_WITHOUT_SUPPORT(UPGRADE_TO_LUCENE_10_0_0, IndexVersionUtils.getPreviousVersion(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT)),
881+
ES_V9X_WITH_SUPPORT(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT, IndexVersion.current());
882+
883+
private final IndexVersion fromVersion;
884+
private final IndexVersion toVersion;
885+
886+
VersionRange(IndexVersion fromVersion, IndexVersion toVersion) {
887+
this.fromVersion = fromVersion;
888+
this.toVersion = toVersion;
889+
}
890+
891+
IndexVersion getRandomVersion() {
892+
// TODO: replace implementation with `IndexVersionUtils::randomVersionBetween` once support is added
893+
// for handling unbalanced version distributions.
894+
NavigableSet<IndexVersion> allReleaseVersions = IndexVersionUtils.allReleasedVersions();
895+
Set<IndexVersion> candidateVersions = allReleaseVersions.subSet(fromVersion, toVersion);
896+
return ESTestCase.randomFrom(candidateVersions);
897+
}
898+
871899
private static final List<WeightedToken> QUERY_VECTORS = Stream.of(RARE_TOKENS, MEDIUM_TOKENS, COMMON_TOKENS)
872900
.flatMap(map -> map.entrySet().stream())
873901
.map(entry -> new WeightedToken(entry.getKey(), entry.getValue()))

0 commit comments

Comments
 (0)