Skip to content

test: fix SparseVectorFieldMapperTests edge case #132937

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Aug 19, 2025
+52 −24
Merged

test: fix SparseVectorFieldMapperTests edge case #132937

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
c1d40b5
test: fix SparseVectorFieldMapperTests edge case
mromaios Aug 14, 2025
7136e09
unmute test, remove unused method
mromaios Aug 14, 2025
d8a53fb
[CI] Auto commit changes from spotless
Aug 14, 2025
1b60a47
add enum for version groups
mromaios Aug 15, 2025
8c01df1
Merge branch '132810_fix_sparse_vector_field_tests' of github.com:mro…
mromaios Aug 15, 2025
fd29bf3
[CI] Auto commit changes from spotless
Aug 15, 2025
aad0ffb
Merge branch 'main' into 132810_fix_sparse_vector_field_tests
mromaios Aug 15, 2025
6e1881c
rename index version range enums
mromaios Aug 18, 2025
3270afb
Merge branch 'main' into 132810_fix_sparse_vector_field_tests
mromaios Aug 18, 2025
3048ab2
[CI] Auto commit changes from spotless
Aug 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions muted-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -489,9 +489,6 @@ tests:
- class: org.elasticsearch.reservedstate.service.FileSettingsServiceIT
method: testSettingsAppliedOnStart
issue: https://github.com/elastic/elasticsearch/issues/131210
- class: org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapperTests
method: testPruningScenarios
issue: https://github.com/elastic/elasticsearch/issues/132810
- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
method: test {p0=search/160_exists_query/Test exists query on mapped date field with no doc values}
issue: https://github.com/elastic/elasticsearch/issues/132828
Expand Down
70 changes: 49 additions & 21 deletions ...er/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import org.elasticsearch.inference.WeightedToken;
import org.elasticsearch.search.lookup.Source;
import org.elasticsearch.search.vectors.SparseVectorQueryWrapper;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.index.IndexVersionUtils;
import org.elasticsearch.xcontent.ToXContent;
import org.elasticsearch.xcontent.XContentBuilder;
Expand All @@ -55,16 +56,16 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.NavigableSet;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.elasticsearch.index.IndexVersions.NEW_SPARSE_VECTOR;
import static org.elasticsearch.index.IndexVersions.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT;
import static org.elasticsearch.index.IndexVersions.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X;
import static org.elasticsearch.index.IndexVersions.UPGRADE_TO_LUCENE_10_0_0;
import static org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper.NEW_SPARSE_VECTOR_INDEX_VERSION;
import static org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper.PREVIOUS_SPARSE_VECTOR_INDEX_VERSION;
import static org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_VERSION;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
import static org.hamcrest.Matchers.containsString;
Expand Down Expand Up @@ -286,7 +287,7 @@ public void testDefaultsWithAndWithoutIncludeDefaultsOlderIndexVersion() throws
IndexVersion indexVersion = IndexVersionUtils.randomVersionBetween(
random(),
UPGRADE_TO_LUCENE_10_0_0,
IndexVersionUtils.getPreviousVersion(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_VERSION)
IndexVersionUtils.getPreviousVersion(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT)
);

XContentBuilder orig = JsonXContent.contentBuilder().startObject();
Expand Down Expand Up @@ -508,21 +509,21 @@ protected IngestScriptSupport ingestScriptSupport() {
@Override
protected String[] getParseMinimalWarnings(IndexVersion indexVersion) {
String[] additionalWarnings = null;
if (indexVersion.before(PREVIOUS_SPARSE_VECTOR_INDEX_VERSION)) {
if (indexVersion.before(IndexVersions.V_8_0_0)) {
additionalWarnings = new String[] { SparseVectorFieldMapper.ERROR_MESSAGE_7X };
}
return Strings.concatStringArrays(super.getParseMinimalWarnings(indexVersion), additionalWarnings);
}

@Override
protected IndexVersion boostNotAllowedIndexVersion() {
return NEW_SPARSE_VECTOR_INDEX_VERSION;
return NEW_SPARSE_VECTOR;
}

public void testSparseVectorUnsupportedIndex() {
IndexVersion version = IndexVersionUtils.randomVersionBetween(
random(),
PREVIOUS_SPARSE_VECTOR_INDEX_VERSION,
IndexVersions.V_8_0_0,
IndexVersions.FIRST_DETACHED_INDEX_VERSION
);
Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(version, fieldMapping(b -> {
Expand Down Expand Up @@ -749,9 +750,11 @@ private void withSearchExecutionContext(MapperService mapperService, CheckedCons

iw.addDocument(mapper.parse(source(b -> b.field("field", RARE_TOKENS))).rootDoc());

// This will lower the averageTokenFreqRatio so that common tokens get pruned with default settings
// This will lower the averageTokenFreqRatio so that common tokens get pruned with default settings.
// Depending on how the index is created, we will have 30-37 numUniqueTokens
// this will result in an averageTokenFreqRatio of 0.1021 - 0.1259
Map<String, Float> uniqueDoc = new TreeMap<>();
for (int i = 0; i < 20; i++) {
for (int i = 0; i < 30; i++) {
uniqueDoc.put("unique" + i, 0.5f);
}
iw.addDocument(mapper.parse(source(b -> b.field("field", uniqueDoc))).rootDoc());
Expand All @@ -765,10 +768,10 @@ private void withSearchExecutionContext(MapperService mapperService, CheckedCons
}

public void testPruningScenarios() throws Exception {
for (int i = 0; i < 120; i++) {
for (int i = 0; i < 200; i++) {
assertPruningScenario(
randomFrom(validIndexPruningScenarios),
new PruningOptions(randomBoolean() ? randomBoolean() : null, randomFrom(PruningConfig.values()))
new PruningOptions(randomFrom(true, false, null), randomFrom(PruningConfig.values()))
);
}
}
Expand Down Expand Up @@ -798,7 +801,8 @@ private PruningScenario getEffectivePruningScenario(
}

if (shouldPrune == null) {
shouldPrune = indexVersion.onOrAfter(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT);
shouldPrune = indexVersion.between(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X, UPGRADE_TO_LUCENE_10_0_0)
|| indexVersion.onOrAfter(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT);
}

PruningScenario pruningScenario = PruningScenario.NO_PRUNING;
Expand Down Expand Up @@ -836,7 +840,7 @@ private List<Query> getExpectedQueryClauses(
}

private void assertPruningScenario(PruningOptions indexPruningOptions, PruningOptions queryPruningOptions) throws IOException {
IndexVersion indexVersion = getIndexVersionForTest(randomBoolean());
IndexVersion indexVersion = getIndexVersion();
MapperService mapperService = createMapperService(indexVersion, getIndexMapping(indexPruningOptions));
PruningScenario effectivePruningScenario = getEffectivePruningScenario(indexPruningOptions, queryPruningOptions, indexVersion);
withSearchExecutionContext(mapperService, (context) -> {
Expand All @@ -855,14 +859,38 @@ private void assertPruningScenario(PruningOptions indexPruningOptions, PruningOp
});
}

private IndexVersion getIndexVersionForTest(boolean usePreviousIndex) {
return usePreviousIndex
? IndexVersionUtils.randomVersionBetween(
random(),
UPGRADE_TO_LUCENE_10_0_0,
IndexVersionUtils.getPreviousVersion(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT)
)
: IndexVersionUtils.randomVersionBetween(random(), SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT, IndexVersion.current());
private static IndexVersion getIndexVersion() {
VersionRange versionRange = randomFrom(VersionRange.values());
return versionRange.getRandomVersion();
}

private enum VersionRange {
ES_V8X_WITHOUT_SUPPORT(
NEW_SPARSE_VECTOR,
IndexVersionUtils.getPreviousVersion(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X)
),
ES_V8X_WITH_SUPPORT(
SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X,
IndexVersionUtils.getPreviousVersion(UPGRADE_TO_LUCENE_10_0_0)
),
ES_V9X_WITHOUT_SUPPORT(UPGRADE_TO_LUCENE_10_0_0, IndexVersionUtils.getPreviousVersion(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT)),
ES_V9X_WITH_SUPPORT(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT, IndexVersion.current());

private final IndexVersion fromVersion;
private final IndexVersion toVersion;

VersionRange(IndexVersion fromVersion, IndexVersion toVersion) {
this.fromVersion = fromVersion;
this.toVersion = toVersion;
}

IndexVersion getRandomVersion() {
// TODO: replace implementation with `IndexVersionUtils::randomVersionBetween` once support is added
// for handling unbalanced version distributions.
NavigableSet<IndexVersion> allReleaseVersions = IndexVersionUtils.allReleasedVersions();
Set<IndexVersion> candidateVersions = allReleaseVersions.subSet(fromVersion, toVersion);
return ESTestCase.randomFrom(candidateVersions);
}
}

private static final List<WeightedToken> QUERY_VECTORS = Stream.of(RARE_TOKENS, MEDIUM_TOKENS, COMMON_TOKENS)
Expand Down