diff --git a/muted-tests.yml b/muted-tests.yml index 68fe2c4f6d9a5..2d6b2892e071d 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -489,9 +489,6 @@ tests: - class: org.elasticsearch.reservedstate.service.FileSettingsServiceIT method: testSettingsAppliedOnStart issue: https://github.com/elastic/elasticsearch/issues/131210 -- class: org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapperTests - method: testPruningScenarios - issue: https://github.com/elastic/elasticsearch/issues/132810 - class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT method: test {p0=search/160_exists_query/Test exists query on mapped date field with no doc values} issue: https://github.com/elastic/elasticsearch/issues/132828 diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapperTests.java index 3d8eb1b454a56..b1647b3fd665e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapperTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.inference.WeightedToken; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.search.vectors.SparseVectorQueryWrapper; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -55,16 +56,16 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.NavigableSet; import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.elasticsearch.index.IndexVersions.NEW_SPARSE_VECTOR; import static org.elasticsearch.index.IndexVersions.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT; +import static org.elasticsearch.index.IndexVersions.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X; import static org.elasticsearch.index.IndexVersions.UPGRADE_TO_LUCENE_10_0_0; -import static org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper.NEW_SPARSE_VECTOR_INDEX_VERSION; -import static org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper.PREVIOUS_SPARSE_VECTOR_INDEX_VERSION; -import static org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_VERSION; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.containsString; @@ -286,7 +287,7 @@ public void testDefaultsWithAndWithoutIncludeDefaultsOlderIndexVersion() throws IndexVersion indexVersion = IndexVersionUtils.randomVersionBetween( random(), UPGRADE_TO_LUCENE_10_0_0, - IndexVersionUtils.getPreviousVersion(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_VERSION) + IndexVersionUtils.getPreviousVersion(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT) ); XContentBuilder orig = JsonXContent.contentBuilder().startObject(); @@ -508,7 +509,7 @@ protected IngestScriptSupport ingestScriptSupport() { @Override protected String[] getParseMinimalWarnings(IndexVersion indexVersion) { String[] additionalWarnings = null; - if (indexVersion.before(PREVIOUS_SPARSE_VECTOR_INDEX_VERSION)) { + if (indexVersion.before(IndexVersions.V_8_0_0)) { additionalWarnings = new String[] { SparseVectorFieldMapper.ERROR_MESSAGE_7X }; } return Strings.concatStringArrays(super.getParseMinimalWarnings(indexVersion), additionalWarnings); @@ -516,13 +517,13 @@ protected String[] getParseMinimalWarnings(IndexVersion indexVersion) { @Override protected IndexVersion boostNotAllowedIndexVersion() { - return NEW_SPARSE_VECTOR_INDEX_VERSION; + return NEW_SPARSE_VECTOR; } public void testSparseVectorUnsupportedIndex() { IndexVersion version = IndexVersionUtils.randomVersionBetween( random(), - PREVIOUS_SPARSE_VECTOR_INDEX_VERSION, + IndexVersions.V_8_0_0, IndexVersions.FIRST_DETACHED_INDEX_VERSION ); Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(version, fieldMapping(b -> { @@ -749,9 +750,11 @@ private void withSearchExecutionContext(MapperService mapperService, CheckedCons iw.addDocument(mapper.parse(source(b -> b.field("field", RARE_TOKENS))).rootDoc()); - // This will lower the averageTokenFreqRatio so that common tokens get pruned with default settings + // This will lower the averageTokenFreqRatio so that common tokens get pruned with default settings. + // Depending on how the index is created, we will have 30-37 numUniqueTokens + // this will result in an averageTokenFreqRatio of 0.1021 - 0.1259 Map uniqueDoc = new TreeMap<>(); - for (int i = 0; i < 20; i++) { + for (int i = 0; i < 30; i++) { uniqueDoc.put("unique" + i, 0.5f); } iw.addDocument(mapper.parse(source(b -> b.field("field", uniqueDoc))).rootDoc()); @@ -765,10 +768,10 @@ private void withSearchExecutionContext(MapperService mapperService, CheckedCons } public void testPruningScenarios() throws Exception { - for (int i = 0; i < 120; i++) { + for (int i = 0; i < 200; i++) { assertPruningScenario( randomFrom(validIndexPruningScenarios), - new PruningOptions(randomBoolean() ? randomBoolean() : null, randomFrom(PruningConfig.values())) + new PruningOptions(randomFrom(true, false, null), randomFrom(PruningConfig.values())) ); } } @@ -798,7 +801,8 @@ private PruningScenario getEffectivePruningScenario( } if (shouldPrune == null) { - shouldPrune = indexVersion.onOrAfter(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT); + shouldPrune = indexVersion.between(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X, UPGRADE_TO_LUCENE_10_0_0) + || indexVersion.onOrAfter(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT); } PruningScenario pruningScenario = PruningScenario.NO_PRUNING; @@ -836,7 +840,7 @@ private List getExpectedQueryClauses( } private void assertPruningScenario(PruningOptions indexPruningOptions, PruningOptions queryPruningOptions) throws IOException { - IndexVersion indexVersion = getIndexVersionForTest(randomBoolean()); + IndexVersion indexVersion = getIndexVersion(); MapperService mapperService = createMapperService(indexVersion, getIndexMapping(indexPruningOptions)); PruningScenario effectivePruningScenario = getEffectivePruningScenario(indexPruningOptions, queryPruningOptions, indexVersion); withSearchExecutionContext(mapperService, (context) -> { @@ -855,14 +859,41 @@ private void assertPruningScenario(PruningOptions indexPruningOptions, PruningOp }); } - private IndexVersion getIndexVersionForTest(boolean usePreviousIndex) { - return usePreviousIndex - ? IndexVersionUtils.randomVersionBetween( - random(), - UPGRADE_TO_LUCENE_10_0_0, - IndexVersionUtils.getPreviousVersion(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT) - ) - : IndexVersionUtils.randomVersionBetween(random(), SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT, IndexVersion.current()); + private static IndexVersion getIndexVersion() { + VersionRange versionRange = randomFrom(VersionRange.values()); + return versionRange.getRandomVersion(); + } + + private enum VersionRange { + ES_V8X_WITHOUT_INDEX_OPTIONS_SUPPORT( + NEW_SPARSE_VECTOR, + IndexVersionUtils.getPreviousVersion(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X) + ), + ES_V8X_WITH_INDEX_OPTIONS_SUPPORT( + SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X, + IndexVersionUtils.getPreviousVersion(UPGRADE_TO_LUCENE_10_0_0) + ), + ES_V9X_WITHOUT_INDEX_OPTIONS_SUPPORT( + UPGRADE_TO_LUCENE_10_0_0, + IndexVersionUtils.getPreviousVersion(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT) + ), + ES_V9X_WITH_INDEX_OPTIONS_SUPPORT(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT, IndexVersion.current()); + + private final IndexVersion fromVersion; + private final IndexVersion toVersion; + + VersionRange(IndexVersion fromVersion, IndexVersion toVersion) { + this.fromVersion = fromVersion; + this.toVersion = toVersion; + } + + IndexVersion getRandomVersion() { + // TODO: replace implementation with `IndexVersionUtils::randomVersionBetween` once support is added + // for handling unbalanced version distributions. + NavigableSet allReleaseVersions = IndexVersionUtils.allReleasedVersions(); + Set candidateVersions = allReleaseVersions.subSet(fromVersion, toVersion); + return ESTestCase.randomFrom(candidateVersions); + } } private static final List QUERY_VECTORS = Stream.of(RARE_TOKENS, MEDIUM_TOKENS, COMMON_TOKENS)