Include v7 in IndexVersionUtils#ALL_VERSIONS (elastic#118793)

We are going to retain read-only compatibility with v7 index versions upon upgrade to v9. This means that all the v7 index versions and corresponding version conditionals will stay around for another major series.

This PR reflects this decision in the codebase. No need to filter index versions when retrieving ALL_VERSIONS, and we can remove corresponding @UpdateForV9 annotations from the IndexVersions class. At the same time, there are tests that need to randomize version but need to write to an index, hence 7x versions should be filtered out.

The overall goal is to extend testing to v7 index versions when possible, and making randomization across writeable versions an exception that certain tests can rely on as needed.

These are the mechanical steps I made in this PR:

- Rename `getFirstVersion` to `getLowestReadCompatibleVersion`, which returns the lowest supported index version ( which can not be written to)
- Introduce `getLowestWriteCompatibleVersion` to identify the lowest writeable version. This is used by tests that used to call `getFirstVersion` and need to write to the index.
- Remove `randomVersion(Random random)` in favour of `randomVersion()` . It was always called providing `random()` which is equivalent to what `randomVersion()` already does.
- Introduce `randomWriteVersion` for tests that need a random writeable version. Moved tests that need it to use it (from `randomVersion` to `randomWriteVersion`)

There is still work to do in `IndexVersionUtils` to extend testing, especially as some randomized tests have a lower bound of `MINIMUM_COMPATIBLE_VERSION` hence don't include v7 index versions yet. We will address that as a follow-up.

Author: javanna

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/NGramTokenizerFactoryTests.java

@@ -161,7 +161,7 @@ public void testBackwardsCompatibilityEdgeNgramTokenFilter() throws Exception {
         for (int i = 0; i < iters; i++) {
             final Index index = new Index("test", "_na_");
             final String name = "ngr";
-            IndexVersion v = IndexVersionUtils.randomVersion(random());
+            IndexVersion v = IndexVersionUtils.randomVersion();
             Builder builder = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3);
             boolean reverse = random().nextBoolean();
             if (reverse) {

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PersianAnalyzerProviderTests.java

@@ -56,7 +56,7 @@ public void testPersianAnalyzerPostLucene10() throws IOException {
     public void testPersianAnalyzerPreLucene10() throws IOException {
         IndexVersion preLucene10Version = IndexVersionUtils.randomVersionBetween(
             random(),
-            IndexVersionUtils.getFirstVersion(),
+            IndexVersionUtils.getLowestReadCompatibleVersion(),
             IndexVersionUtils.getPreviousVersion(IndexVersions.UPGRADE_TO_LUCENE_10_0_0)
         );
         Settings settings = ESTestCase.indexSettings(1, 1)

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RomanianAnalyzerTests.java

@@ -57,7 +57,7 @@ public void testRomanianAnalyzerPostLucene10() throws IOException {
     public void testRomanianAnalyzerPreLucene10() throws IOException {
         IndexVersion preLucene10Version = IndexVersionUtils.randomVersionBetween(
             random(),
-            IndexVersionUtils.getFirstVersion(),
+            IndexVersionUtils.getLowestReadCompatibleVersion(),
             IndexVersionUtils.getPreviousVersion(IndexVersions.UPGRADE_TO_LUCENE_10_0_0)
         );
         Settings settings = ESTestCase.indexSettings(1, 1)

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java

@@ -39,7 +39,7 @@ public class StemmerTokenFilterFactoryTests extends ESTokenStreamTestCase {
     public void testEnglishFilterFactory() throws IOException {
         int iters = scaledRandomIntBetween(20, 100);
         for (int i = 0; i < iters; i++) {
-            IndexVersion v = IndexVersionUtils.randomVersion(random());
+            IndexVersion v = IndexVersionUtils.randomVersion();
             Settings settings = Settings.builder()
                 .put("index.analysis.filter.my_english.type", "stemmer")
                 .put("index.analysis.filter.my_english.language", "english")
@@ -66,7 +66,7 @@ public void testPorter2FilterFactory() throws IOException {
         int iters = scaledRandomIntBetween(20, 100);
         for (int i = 0; i < iters; i++) {
 
-            IndexVersion v = IndexVersionUtils.randomVersion(random());
+            IndexVersion v = IndexVersionUtils.randomVersion();
             Settings settings = Settings.builder()
                 .put("index.analysis.filter.my_porter2.type", "stemmer")
                 .put("index.analysis.filter.my_porter2.language", "porter2")
@@ -90,7 +90,7 @@ public void testPorter2FilterFactory() throws IOException {
     }
 
     public void testMultipleLanguagesThrowsException() throws IOException {
-        IndexVersion v = IndexVersionUtils.randomVersion(random());
+        IndexVersion v = IndexVersionUtils.randomVersion();
         Settings settings = Settings.builder()
             .put("index.analysis.filter.my_english.type", "stemmer")
             .putList("index.analysis.filter.my_english.language", "english", "light_english")
@@ -142,7 +142,7 @@ private static Analyzer createGermanStemmer(String variant, IndexVersion v) thro
     }
 
     public void testKpDeprecation() throws IOException {
-        IndexVersion v = IndexVersionUtils.randomVersion(random());
+        IndexVersion v = IndexVersionUtils.randomVersion();
         Settings settings = Settings.builder()
             .put("index.analysis.filter.my_kp.type", "stemmer")
             .put("index.analysis.filter.my_kp.language", "kp")
@@ -155,7 +155,7 @@ public void testKpDeprecation() throws IOException {
     }
 
     public void testLovinsDeprecation() throws IOException {
-        IndexVersion v = IndexVersionUtils.randomVersion(random());
+        IndexVersion v = IndexVersionUtils.randomVersion();
         Settings settings = Settings.builder()
             .put("index.analysis.filter.my_lovins.type", "stemmer")
             .put("index.analysis.filter.my_lovins.language", "lovins")

server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java

@@ -236,7 +236,7 @@ private static IndexMetadata indexMetadata(final Client client, final String ind
 
     public void testCreateShrinkIndex() {
         internalCluster().ensureAtLeastNumDataNodes(2);
-        IndexVersion version = IndexVersionUtils.randomVersion(random());
+        IndexVersion version = IndexVersionUtils.randomWriteVersion();
         prepareCreate("source").setSettings(
             Settings.builder().put(indexSettings()).put("number_of_shards", randomIntBetween(2, 7)).put("index.version.created", version)
         ).get();

server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java

@@ -71,6 +71,7 @@
 import static org.elasticsearch.test.XContentTestUtils.convertToMap;
 import static org.elasticsearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder;
 import static org.elasticsearch.test.index.IndexVersionUtils.randomVersion;
+import static org.elasticsearch.test.index.IndexVersionUtils.randomWriteVersion;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
@@ -218,7 +219,7 @@ private ClusterState.Builder randomCoordinationMetadata(ClusterState clusterStat
 
     private DiscoveryNode randomNode(String nodeId) {
         Version nodeVersion = VersionUtils.randomVersion(random());
-        IndexVersion indexVersion = randomVersion(random());
+        IndexVersion indexVersion = randomVersion();
         return DiscoveryNodeUtils.builder(nodeId)
             .roles(emptySet())
             .version(nodeVersion, IndexVersion.fromId(indexVersion.id() - 1_000_000), indexVersion)
@@ -561,7 +562,7 @@ public IndexMetadata randomCreate(String name) {
                 IndexMetadata.Builder builder = IndexMetadata.builder(name);
                 Settings.Builder settingsBuilder = Settings.builder();
                 setRandomIndexSettings(random(), settingsBuilder);
-                settingsBuilder.put(randomSettings(Settings.EMPTY)).put(IndexMetadata.SETTING_VERSION_CREATED, randomVersion(random()));
+                settingsBuilder.put(randomSettings(Settings.EMPTY)).put(IndexMetadata.SETTING_VERSION_CREATED, randomWriteVersion());
                 builder.settings(settingsBuilder);
                 builder.numberOfShards(randomIntBetween(1, 10)).numberOfReplicas(randomInt(10));
                 builder.eventIngestedRange(IndexLongFieldRange.UNKNOWN, TransportVersion.current());
@@ -736,7 +737,7 @@ public ClusterState.Custom randomCreate(String name) {
                             ImmutableOpenMap.of(),
                             null,
                             SnapshotInfoTestUtils.randomUserMetadata(),
-                            randomVersion(random())
+                            randomVersion()
                         )
                     );
                     case 1 -> new RestoreInProgress.Builder().add(

server/src/internalClusterTest/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java

@@ -47,7 +47,7 @@ public void testThatPreBuiltAnalyzersAreNotClosedOnIndexClose() throws Exception
             PreBuiltAnalyzers preBuiltAnalyzer = PreBuiltAnalyzers.values()[randomInt];
             String name = preBuiltAnalyzer.name().toLowerCase(Locale.ROOT);
 
-            IndexVersion randomVersion = IndexVersionUtils.randomVersion(random());
+            IndexVersion randomVersion = IndexVersionUtils.randomWriteVersion();
             if (loadedAnalyzers.containsKey(preBuiltAnalyzer) == false) {
                 loadedAnalyzers.put(preBuiltAnalyzer, new ArrayList<>());
             }

server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java

@@ -2050,7 +2050,7 @@ public void testPostRecoveryMergeDisabledOnOlderIndices() throws Exception {
                     IndexMetadata.SETTING_VERSION_CREATED,
                     IndexVersionUtils.randomVersionBetween(
                         random(),
-                        IndexVersionUtils.getFirstVersion(),
+                        IndexVersionUtils.getLowestWriteCompatibleVersion(),
                         IndexVersionUtils.getPreviousVersion(IndexVersions.MERGE_ON_RECOVERY_VERSION)
                     )
                 )

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -12,7 +12,6 @@
 import org.apache.lucene.util.Version;
 import org.elasticsearch.ReleaseVersions;
 import org.elasticsearch.core.Assertions;
-import org.elasticsearch.core.UpdateForV9;
 
 import java.lang.reflect.Field;
 import java.text.ParseException;
@@ -25,6 +24,7 @@
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.function.IntFunction;
+import java.util.stream.Collectors;
 
 @SuppressWarnings("deprecation")
 public class IndexVersions {
@@ -58,7 +58,6 @@ private static Version parseUnchecked(String version) {
         }
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_FOUNDATIONS) // remove the index versions with which v9 will not need to interact
     public static final IndexVersion ZERO = def(0, Version.LATEST);
 
     public static final IndexVersion V_7_0_0 = def(7_00_00_99, parseUnchecked("8.0.0"));
@@ -244,10 +243,12 @@ static NavigableMap<Integer, IndexVersion> getAllVersionIds(Class<?> cls) {
         return Collections.unmodifiableNavigableMap(builder);
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
-    // We can simplify this once we've removed all references to index versions earlier than MINIMUM_COMPATIBLE
+    static Collection<IndexVersion> getAllWriteVersions() {
+        return VERSION_IDS.values().stream().filter(v -> v.onOrAfter(IndexVersions.MINIMUM_COMPATIBLE)).collect(Collectors.toSet());
+    }
+
     static Collection<IndexVersion> getAllVersions() {
-        return VERSION_IDS.values().stream().filter(v -> v.onOrAfter(MINIMUM_COMPATIBLE)).toList();
+        return VERSION_IDS.values();
     }
 
     static final IntFunction<String> VERSION_LOOKUP = ReleaseVersions.generateVersionsLookup(IndexVersions.class, LATEST_DEFINED.id());

server/src/test/java/org/elasticsearch/cluster/metadata/HumanReadableIndexSettingsTests.java

@@ -22,7 +22,7 @@
 
 public class HumanReadableIndexSettingsTests extends ESTestCase {
     public void testHumanReadableSettings() {
-        IndexVersion versionCreated = randomVersion(random());
+        IndexVersion versionCreated = randomVersion();
         long created = System.currentTimeMillis();
         Settings testSettings = Settings.builder()
             .put(IndexMetadata.SETTING_VERSION_CREATED, versionCreated)