Only use NavigableSet for test uses

Introduce logarithmic random algorithm

Author: thecoop

server/src/main/java/org/elasticsearch/TransportVersion.java

@@ -18,10 +18,9 @@
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
-import java.util.NavigableSet;
 import java.util.ServiceLoader;
-import java.util.TreeSet;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -113,9 +112,9 @@ public static TransportVersion current() {
     }
 
     /**
-     * Sorted set of all defined transport versions
+     * Sorted list of all defined transport versions
      */
-    public static NavigableSet<TransportVersion> getAllVersions() {
+    public static List<TransportVersion> getAllVersions() {
         return VersionsHolder.ALL_VERSIONS;
     }
 
@@ -161,7 +160,7 @@ public String toString() {
     }
 
     private static class VersionsHolder {
-        private static final NavigableSet<TransportVersion> ALL_VERSIONS;
+        private static final List<TransportVersion> ALL_VERSIONS;
         private static final Map<Integer, TransportVersion> ALL_VERSIONS_MAP;
         private static final TransportVersion CURRENT;
 
@@ -173,8 +172,7 @@ private static class VersionsHolder {
             if (extendedVersions.isEmpty()) {
                 ALL_VERSIONS = TransportVersions.DEFINED_VERSIONS;
             } else {
-                ALL_VERSIONS = Stream.concat(TransportVersions.DEFINED_VERSIONS.stream(), extendedVersions.stream())
-                    .collect(Collectors.toCollection(TreeSet::new));
+                ALL_VERSIONS = Stream.concat(TransportVersions.DEFINED_VERSIONS.stream(), extendedVersions.stream()).sorted().toList();
             }
 
             ALL_VERSIONS_MAP = ALL_VERSIONS.stream().collect(Collectors.toUnmodifiableMap(TransportVersion::id, Function.identity()));

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -13,10 +13,11 @@
 import org.elasticsearch.core.UpdateForV9;
 
 import java.lang.reflect.Field;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
-import java.util.NavigableSet;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.function.IntFunction;
@@ -241,7 +242,7 @@ static TransportVersion def(int id) {
     /**
      * Sorted list of all versions defined in this class
      */
-    static final NavigableSet<TransportVersion> DEFINED_VERSIONS = collectAllVersionIdsDefinedInClass(TransportVersions.class);
+    static final List<TransportVersion> DEFINED_VERSIONS = collectAllVersionIdsDefinedInClass(TransportVersions.class);
 
     // the highest transport version constant defined
     static final TransportVersion LATEST_DEFINED;
@@ -253,9 +254,9 @@ static TransportVersion def(int id) {
         IDS = null;
     }
 
-    public static NavigableSet<TransportVersion> collectAllVersionIdsDefinedInClass(Class<?> cls) {
+    public static List<TransportVersion> collectAllVersionIdsDefinedInClass(Class<?> cls) {
         Map<Integer, String> versionIdFields = new HashMap<>();
-        NavigableSet<TransportVersion> definedTransportVersions = new TreeSet<>();
+        List<TransportVersion> definedTransportVersions = new ArrayList<>();
 
         Set<String> ignore = Set.of("ZERO", "CURRENT", "MINIMUM_COMPATIBLE", "MINIMUM_CCS_VERSION");
 
@@ -289,7 +290,8 @@ public static NavigableSet<TransportVersion> collectAllVersionIdsDefinedInClass(
             }
         }
 
-        return Collections.unmodifiableNavigableSet(definedTransportVersions);
+        Collections.sort(definedTransportVersions);
+        return List.copyOf(definedTransportVersions);
     }
 
     static final IntFunction<String> VERSION_LOOKUP = ReleaseVersions.generateVersionsLookup(TransportVersions.class, LATEST_DEFINED.id());

server/src/main/java/org/elasticsearch/Version.java

@@ -23,15 +23,14 @@
 import java.io.IOException;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.NavigableMap;
-import java.util.NavigableSet;
 import java.util.TreeMap;
-import java.util.TreeSet;
 
 public class Version implements VersionId<Version>, ToXContentFragment {
     /*
@@ -495,9 +494,9 @@ public int hashCode() {
      * The argument would normally be Version.class but is exposed for
      * testing with other classes-containing-version-constants.
      */
-    public static NavigableSet<Version> getDeclaredVersions(final Class<?> versionClass) {
+    public static List<Version> getDeclaredVersions(final Class<?> versionClass) {
         final Field[] fields = versionClass.getFields();
-        final TreeSet<Version> versions = new TreeSet<>();
+        final List<Version> versions = new ArrayList<>();
         for (final Field field : fields) {
             final int mod = field.getModifiers();
             if (false == Modifier.isStatic(mod) && Modifier.isFinal(mod) && Modifier.isPublic(mod)) {
@@ -518,6 +517,7 @@ public static NavigableSet<Version> getDeclaredVersions(final Class<?> versionCl
                 throw new RuntimeException(e);
             }
         }
+        Collections.sort(versions);
         return versions;
     }
 }

test/framework/src/main/java/org/elasticsearch/test/BWCVersions.java

@@ -18,7 +18,7 @@ public final class BWCVersions {
     private BWCVersions() {}
 
     public static NavigableSet<TransportVersion> getAllBWCVersions() {
-        return TransportVersion.getAllVersions().tailSet(TransportVersions.MINIMUM_COMPATIBLE, true);
+        return TransportVersionUtils.allReleasedVersions().tailSet(TransportVersions.MINIMUM_COMPATIBLE, true);
     }
 
     public static final NavigableSet<TransportVersion> DEFAULT_BWC_VERSIONS = getAllBWCVersions();

test/framework/src/main/java/org/elasticsearch/test/TransportVersionUtils.java

@@ -13,15 +13,24 @@
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.core.Nullable;
 
+import java.util.Collections;
 import java.util.NavigableSet;
 import java.util.Random;
 import java.util.Set;
+import java.util.TreeSet;
 import java.util.stream.Collectors;
 
+import static org.apache.lucene.tests.util.LuceneTestCase.random;
+
 public class TransportVersionUtils {
+
+    private static final NavigableSet<TransportVersion> RELEASED_VERSIONS = Collections.unmodifiableNavigableSet(
+        new TreeSet<>(TransportVersion.getAllVersions())
+    );
+
     /** Returns all released versions */
     public static NavigableSet<TransportVersion> allReleasedVersions() {
-        return TransportVersion.getAllVersions();
+        return RELEASED_VERSIONS;
     }
 
     /** Returns the oldest known {@link TransportVersion} */
@@ -31,7 +40,7 @@ public static TransportVersion getFirstVersion() {
 
     /** Returns a random {@link TransportVersion} from all available versions. */
     public static TransportVersion randomVersion() {
-        return ESTestCase.randomFrom(allReleasedVersions());
+        return VersionUtils.randomFrom(random(), allReleasedVersions(), TransportVersion::fromId);
     }
 
     /** Returns a random {@link TransportVersion} from all available versions without the ignore set */
@@ -41,7 +50,7 @@ public static TransportVersion randomVersion(Set<TransportVersion> ignore) {
 
     /** Returns a random {@link TransportVersion} from all available versions. */
     public static TransportVersion randomVersion(Random random) {
-        return ESTestCase.randomFrom(random, allReleasedVersions());
+        return VersionUtils.randomFrom(random, allReleasedVersions(), TransportVersion::fromId);
     }
 
     /** Returns a random {@link TransportVersion} between <code>minVersion</code> and <code>maxVersion</code> (inclusive). */
@@ -68,7 +77,7 @@ public static TransportVersion randomVersionBetween(
             versions = versions.headSet(maxVersion, true);
         }
 
-        return ESTestCase.randomFrom(random, versions);
+        return VersionUtils.randomFrom(random, versions, TransportVersion::fromId);
     }
 
     public static TransportVersion getPreviousVersion() {
@@ -78,28 +87,28 @@ public static TransportVersion getPreviousVersion() {
     }
 
     public static TransportVersion getPreviousVersion(TransportVersion version) {
-        var versions = allReleasedVersions().headSet(version, false);
-        if (versions.isEmpty()) {
+        TransportVersion lower = allReleasedVersions().lower(version);
+        if (lower == null) {
             throw new IllegalArgumentException("couldn't find any released versions before [" + version + "]");
         }
-        return versions.getLast();
+        return lower;
     }
 
     public static TransportVersion getNextVersion(TransportVersion version) {
         return getNextVersion(version, false);
     }
 
     public static TransportVersion getNextVersion(TransportVersion version, boolean createIfNecessary) {
-        var versions = allReleasedVersions().tailSet(version, false);
-        if (versions.isEmpty()) {
+        TransportVersion higher = allReleasedVersions().higher(version);
+        if (higher == null) {
             if (createIfNecessary) {
                 // create a new transport version one greater than specified
                 return new TransportVersion(version.id() + 1);
             } else {
                 throw new IllegalArgumentException("couldn't find any released versions after [" + version + "]");
             }
         }
-        return versions.getFirst();
+        return higher;
     }
 
     /** Returns a random {@code TransportVersion} that is compatible with {@link TransportVersion#current()} */

test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java

@@ -9,18 +9,26 @@
 
 package org.elasticsearch.test;
 
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
+
 import org.elasticsearch.Build;
 import org.elasticsearch.Version;
+import org.elasticsearch.common.VersionId;
 import org.elasticsearch.core.Nullable;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.NavigableSet;
 import java.util.Random;
+import java.util.TreeSet;
+import java.util.function.IntFunction;
 
 /** Utilities for selecting versions in tests */
 public class VersionUtils {
 
-    private static final NavigableSet<Version> ALL_VERSIONS = Version.getDeclaredVersions(Version.class);
+    private static final NavigableSet<Version> ALL_VERSIONS = Collections.unmodifiableNavigableSet(
+        new TreeSet<>(Version.getDeclaredVersions(Version.class))
+    );
 
     /**
      * Returns an immutable, sorted list containing all versions, both released and unreleased.
@@ -69,7 +77,7 @@ public static Version getFirstVersion() {
 
     /** Returns a random {@link Version} from all available versions. */
     public static Version randomVersion(Random random) {
-        return ESTestCase.randomFrom(random, ALL_VERSIONS);
+        return randomFrom(random, ALL_VERSIONS, Version::fromId);
     }
 
     /** Returns a random {@link Version} from all available versions, that is compatible with the given version. */
@@ -98,11 +106,24 @@ public static Version randomVersionBetween(Random random, @Nullable Version minV
             versions = versions.headSet(maxVersion, true);
         }
 
-        return ESTestCase.randomFrom(random, versions);
+        return randomFrom(random, versions, Version::fromId);
     }
 
     /** Returns the maximum {@link Version} that is compatible with the given version. */
     public static Version maxCompatibleVersion(Version version) {
-        return ALL_VERSIONS.descendingSet().stream().filter(version::isCompatible).filter(version::onOrBefore).findFirst().orElseThrow();
+        return ALL_VERSIONS.tailSet(version, true).descendingSet().stream().filter(version::isCompatible).findFirst().orElseThrow();
+    }
+
+    public static <T extends VersionId<T>> T randomFrom(Random random, NavigableSet<T> set, IntFunction<T> ctor) {
+        // get the first and last id, pick a random id in the middle, then find that id in the set in O(nlogn) time
+        // this assumes the id numbers are reasonably evenly distributed in the set
+        assert set.isEmpty() == false;
+        int lowest = set.getFirst().id();
+        int highest = set.getLast().id();
+
+        T randomId = ctor.apply(RandomNumbers.randomIntBetween(random, lowest, highest));
+        // try to find the id below, then the id above. We're just looking for *some* item in the set that is close to randomId
+        T found = set.floor(randomId);
+        return found != null ? found : set.ceiling(randomId);
     }
 }

test/framework/src/main/java/org/elasticsearch/test/index/IndexVersionUtils.java

@@ -14,12 +14,15 @@
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.KnownIndexVersions;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
 
 import java.util.NavigableSet;
 import java.util.Random;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.apache.lucene.tests.util.LuceneTestCase.random;
+
 public class IndexVersionUtils {
 
     private static final NavigableSet<IndexVersion> ALL_VERSIONS = KnownIndexVersions.ALL_VERSIONS;
@@ -42,12 +45,12 @@ public static IndexVersion getLowestWriteCompatibleVersion() {
 
     /** Returns a random {@link IndexVersion} from all available versions. */
     public static IndexVersion randomVersion() {
-        return ESTestCase.randomFrom(ALL_VERSIONS);
+        return VersionUtils.randomFrom(random(), ALL_VERSIONS, IndexVersion::fromId);
     }
 
     /** Returns a random {@link IndexVersion} from all versions that can be written to. */
     public static IndexVersion randomWriteVersion() {
-        return ESTestCase.randomFrom(ALL_WRITE_VERSIONS);
+        return VersionUtils.randomFrom(random(), ALL_WRITE_VERSIONS, IndexVersion::fromId);
     }
 
     /** Returns a random {@link IndexVersion} from all available versions without the ignore set */
@@ -75,7 +78,7 @@ public static IndexVersion randomVersionBetween(Random random, @Nullable IndexVe
             versions = versions.headSet(maxVersion, true);
         }
 
-        return ESTestCase.randomFrom(random, versions);
+        return VersionUtils.randomFrom(random, versions, IndexVersion::fromId);
     }
 
     public static IndexVersion getPreviousVersion() {
@@ -85,23 +88,23 @@ public static IndexVersion getPreviousVersion() {
     }
 
     public static IndexVersion getPreviousVersion(IndexVersion version) {
-        var versions = allReleasedVersions().headSet(version, false);
-        if (versions.isEmpty()) {
+        IndexVersion lower = allReleasedVersions().lower(version);
+        if (lower == null) {
             throw new IllegalArgumentException("couldn't find any released versions before [" + version + "]");
         }
-        return versions.getLast();
+        return lower;
     }
 
     public static IndexVersion getPreviousMajorVersion(IndexVersion version) {
         return IndexVersion.getMinimumCompatibleIndexVersion(version.id());
     }
 
     public static IndexVersion getNextVersion(IndexVersion version) {
-        var versions = allReleasedVersions().tailSet(version, false);
-        if (versions.isEmpty()) {
+        IndexVersion higher = allReleasedVersions().higher(version);
+        if (higher == null) {
             throw new IllegalArgumentException("couldn't find any released versions after [" + version + "]");
         }
-        return versions.getFirst();
+        return higher;
     }
 
     /** Returns a random {@code IndexVersion} that is compatible with {@link IndexVersion#current()} */

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/AbstractBWCSerializationTestCase.java

@@ -10,6 +10,7 @@
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
@@ -20,7 +21,7 @@
 public abstract class AbstractBWCSerializationTestCase<T extends Writeable & ToXContent> extends AbstractXContentSerializingTestCase<T> {
 
     private static NavigableSet<TransportVersion> getAllBWCVersions() {
-        return TransportVersion.getAllVersions().tailSet(TransportVersions.MINIMUM_COMPATIBLE, true);
+        return TransportVersionUtils.allReleasedVersions().tailSet(TransportVersions.MINIMUM_COMPATIBLE, true);
     }
 
     private static final NavigableSet<TransportVersion> DEFAULT_BWC_VERSIONS = getAllBWCVersions();

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/AbstractBWCWireSerializingTestCase.java

@@ -10,6 +10,7 @@
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.test.TransportVersionUtils;
 
 import java.io.IOException;
 import java.util.NavigableSet;
@@ -19,7 +20,7 @@
 public abstract class AbstractBWCWireSerializingTestCase<T extends Writeable> extends AbstractWireSerializingTestCase<T> {
 
     private static NavigableSet<TransportVersion> getAllBWCVersions() {
-        return TransportVersion.getAllVersions().tailSet(TransportVersions.MINIMUM_COMPATIBLE, true);
+        return TransportVersionUtils.allReleasedVersions().tailSet(TransportVersions.MINIMUM_COMPATIBLE, true);
     }
 
     private static final NavigableSet<TransportVersion> DEFAULT_BWC_VERSIONS = getAllBWCVersions();