Random overhead improvements (#15327)

* Fix the code so that testing with -Ptests.asserts=false passes.

* More tests that don't work with assertions disabled.

* Pick tests.asserts randomly based on the main seed. Use assertions 75% of the time.

* Changes entry.

* Clean up duplicate assertion-verifying code.

* Add a new build option tests.random.maxcalls for debugging too many calls to the asserting Random returned from LuceneTestCase.random(). Added LuceneTestCase.nonAssertingRandom(Random) to acquire a fast, non-asserting random instance for intense local computation. Modified a bunch of tests that exceeded 1 million calls to random() result.

* Added another option for low-level debugging of intense random() calls. tests.random.maxacquires will terminate if too many random() calls are called from within a single test. Corrected a few tests where this was the case.

* Tidy and correct a debugging check in LuceneTestCase.

* Final minor touches.

* Changes entry.

* Changes entry.

Author: dweiss

build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.java

@@ -221,6 +221,15 @@ public void apply(Project project) {
     buildOptions.addIntOption("tests.timeoutSuite", "Timeout (in millis) for an entire suite.");
     optionsInheritedAsProperties.add("tests.timeoutSuite");
 
+    buildOptions.addIntOption(
+        "tests.random.maxcalls",
+        "Max number of calls to Randoms returned by LuceneTestCase.random()");
+    optionsInheritedAsProperties.add("tests.random.maxcalls");
+
+    buildOptions.addIntOption(
+        "tests.random.maxacquires", "Max number of per-test calls to LuceneTestCase.random()");
+    optionsInheritedAsProperties.add("tests.random.maxacquires");
+
     Provider<Boolean> assertsOption =
         buildOptions.addBooleanOption(
             "tests.asserts",

lucene/CHANGES.txt

@@ -63,8 +63,6 @@ Improvements
   This avoids clashes with other ASM versions on classpath because it removes the dependency.
   (Uwe Schindler)
 
-
-
 * GITHUB#15002: Remove synchronized WeakHashMap from IndexReader. This was added to help prevent
   SIGSEGV with the old unsafe "mmap hack", which has been replaced by MemorySegments.
   (Uwe Schindler, Robert Muir)
@@ -106,7 +104,11 @@ Changes in Runtime Behavior
 
 Build
 ---------------------
-* GITHUB#15325: Run the tests with assertions disabled ~25% of the time #15325 (Dawid Weiss)
+* GITHUB#15327: New low-level build options to detect abuse of LuceneTestCase.random():
+  tests.random.maxacquires and tests.random.maxcalls (Robert Muir, Dawid Weiss)
+
+* GITHUB#15325: Run the tests with assertions disabled ~25% of the time #15325
+  (Robert Muir, Dawid Weiss)
 
 * GITHUB#14804: Detect and ban wildcard imports in Java (Robert Muir, Dawid Weiss)
 

lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestRollingCharBuffer.java

@@ -28,7 +28,7 @@ public void test() throws Exception {
 
     RollingCharBuffer buffer = new RollingCharBuffer();
 
-    Random random = random();
+    Random random = nonAssertingRandom(random());
     for (int iter = 0; iter < ITERS; iter++) {
       final int stringLen = random.nextBoolean() ? random.nextInt(50) : random.nextInt(20000);
       final String s;

lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/compressing/AbstractTestCompressionMode.java

@@ -39,6 +39,7 @@ static byte[] randomArray(Random random) {
   }
 
   static byte[] randomArray(Random random, int length, int max) {
+    random = nonAssertingRandom(random);
     final byte[] arr = new byte[length];
     for (int i = 0; i < arr.length; ++i) {
       arr[i] = (byte) RandomNumbers.randomIntBetween(random, 0, max);
@@ -83,7 +84,7 @@ byte[] decompress(byte[] compressed, int originalLength, int offset, int length)
   }
 
   public void testDecompress() throws IOException {
-    Random random = random();
+    Random random = nonAssertingRandom(random());
     final int iterations = atLeast(random, 3);
     for (int i = 0; i < iterations; ++i) {
       final byte[] decompressed = randomArray(random);
@@ -99,7 +100,7 @@ public void testDecompress() throws IOException {
   }
 
   public void testPartialDecompress() throws IOException {
-    Random random = random();
+    Random random = nonAssertingRandom(random());
     final int iterations = atLeast(random, 3);
     for (int i = 0; i < iterations; ++i) {
       final byte[] decompressed = randomArray(random);

lucene/classification/src/test/org/apache/lucene/classification/utils/TestDataSplitter.java

@@ -63,7 +63,7 @@ public void setUp() throws Exception {
     ft.setStoreTermVectorPositions(true);
 
     Document doc;
-    Random rnd = random();
+    Random rnd = nonAssertingRandom(random());
     int numDocs = atLeast(100);
     for (int i = 0; i < numDocs; i++) {
       doc = new Document();

lucene/core/src/test/org/apache/lucene/analysis/standard/TestStandardAnalyzer.java

@@ -76,19 +76,20 @@ public void testLargePartiallyMatchingToken() throws Exception {
     };
 
     StringBuilder builder = new StringBuilder();
-    int numChars = TestUtil.nextInt(random(), 100 * 1024, 1024 * 1024);
+    var rnd = nonAssertingRandom(random());
+    int numChars = TestUtil.nextInt(rnd, 100 * 1024, 1024 * 1024);
     for (int i = 0; i < numChars; ) {
       builder.append(
-          WordBreak_ExtendNumLet_chars[random().nextInt(WordBreak_ExtendNumLet_chars.length)]);
+          WordBreak_ExtendNumLet_chars[rnd.nextInt(WordBreak_ExtendNumLet_chars.length)]);
       ++i;
-      if (random().nextBoolean()) {
-        int numFormatExtendChars = TestUtil.nextInt(random(), 1, 8);
+      if (rnd.nextBoolean()) {
+        int numFormatExtendChars = TestUtil.nextInt(rnd, 1, 8);
         for (int j = 0; j < numFormatExtendChars; ++j) {
           int codepoint;
-          if (random().nextBoolean()) {
-            codepoint = WordBreak_Format_chars[random().nextInt(WordBreak_Format_chars.length)];
+          if (rnd.nextBoolean()) {
+            codepoint = WordBreak_Format_chars[rnd.nextInt(WordBreak_Format_chars.length)];
           } else {
-            codepoint = WordBreak_Extend_chars[random().nextInt(WordBreak_Extend_chars.length)];
+            codepoint = WordBreak_Extend_chars[rnd.nextInt(WordBreak_Extend_chars.length)];
           }
           char[] chars = Character.toChars(codepoint);
           builder.append(chars);

lucene/core/src/test/org/apache/lucene/codecs/lucene103/blocktree/TestTrie.java

@@ -194,9 +194,10 @@ private void testTrieLookup(Supplier<byte[]> randomBytesSupplier, int round) thr
   }
 
   private static byte[] randomBytes() {
-    byte[] bytes = new byte[random().nextInt(256) + 1];
+    var random = nonAssertingRandom(random());
+    byte[] bytes = new byte[random.nextInt(256) + 1];
     for (int i = 1; i < bytes.length; i++) {
-      bytes[i] = (byte) random().nextInt(1 << (i % 9));
+      bytes[i] = (byte) random.nextInt(1 << (i % 9));
     }
     return bytes;
   }

lucene/core/src/test/org/apache/lucene/geo/TestGeoEncodingUtils.java

@@ -27,6 +27,7 @@
 import static org.apache.lucene.geo.GeoUtils.MIN_LAT_INCL;
 import static org.apache.lucene.geo.GeoUtils.MIN_LON_INCL;
 
+import com.carrotsearch.randomizedtesting.RandomizedTest;
 import java.util.Random;
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.tests.util.TestUtil;
@@ -42,8 +43,8 @@ public class TestGeoEncodingUtils extends LuceneTestCase {
    */
   public void testLatitudeQuantization() throws Exception {
     final double LATITUDE_DECODE = 180.0D / (0x1L << 32);
-    Random random = random();
-    for (int i = 0; i < 10000; i++) {
+    Random random = nonAssertingRandom(random());
+    for (int i = 0; i < RandomizedTest.randomIntBetween(1000, 10000); i++) {
       int encoded = random.nextInt();
       double min = MIN_LAT_INCL + (encoded - (long) Integer.MIN_VALUE) * LATITUDE_DECODE;
       double decoded = decodeLatitude(encoded);
@@ -92,8 +93,8 @@ public void testLatitudeQuantization() throws Exception {
    */
   public void testLongitudeQuantization() throws Exception {
     final double LONGITUDE_DECODE = 360.0D / (0x1L << 32);
-    Random random = random();
-    for (int i = 0; i < 10000; i++) {
+    Random random = nonAssertingRandom(random());
+    for (int i = 0; i < RandomizedTest.randomIntBetween(1000, 10000); i++) {
       int encoded = random.nextInt();
       double min = MIN_LON_INCL + (encoded - (long) Integer.MIN_VALUE) * LONGITUDE_DECODE;
       double decoded = decodeLongitude(encoded);

lucene/core/src/test/org/apache/lucene/geo/TestXYRectangle.java

@@ -139,17 +139,19 @@ public void testEqualsAndHashCode() {
 
   /** make sure that if a point is inside a circle, it is inside of the bbox as well */
   public void testRandomCircleToBBox() {
+    var random = nonAssertingRandom(random());
+
     int iters = atLeast(100);
     for (int iter = 0; iter < iters; iter++) {
 
-      float centerX = ShapeTestUtil.nextFloat(random());
-      float centerY = ShapeTestUtil.nextFloat(random());
+      float centerX = ShapeTestUtil.nextFloat(random);
+      float centerY = ShapeTestUtil.nextFloat(random);
 
       final float radius;
-      if (random().nextBoolean()) {
-        radius = random().nextFloat() * TestUtil.nextInt(random(), 1, 100000);
+      if (random.nextBoolean()) {
+        radius = random.nextFloat() * TestUtil.nextInt(random, 1, 100000);
       } else {
-        radius = Math.abs(ShapeTestUtil.nextFloat(random()));
+        radius = Math.abs(ShapeTestUtil.nextFloat(random));
       }
 
       XYRectangle bbox = XYRectangle.fromPointDistance(centerX, centerY, radius);
@@ -159,16 +161,16 @@ public void testRandomCircleToBBox() {
       for (int i = 0; i < numPointsToTry; i++) {
 
         double x;
-        if (random().nextBoolean()) {
-          x = Math.min(Float.MAX_VALUE, centerX + radius + random().nextDouble());
+        if (random.nextBoolean()) {
+          x = Math.min(Float.MAX_VALUE, centerX + radius + random.nextDouble());
         } else {
-          x = Math.max(-Float.MAX_VALUE, centerX + radius - random().nextDouble());
+          x = Math.max(-Float.MAX_VALUE, centerX + radius - random.nextDouble());
         }
         double y;
-        if (random().nextBoolean()) {
-          y = Math.min(Float.MAX_VALUE, centerY + radius + random().nextDouble());
+        if (random.nextBoolean()) {
+          y = Math.min(Float.MAX_VALUE, centerY + radius + random.nextDouble());
         } else {
-          y = Math.max(-Float.MAX_VALUE, centerY + radius - random().nextDouble());
+          y = Math.max(-Float.MAX_VALUE, centerY + radius - random.nextDouble());
         }
 
         // cartesian says it's within the circle:

lucene/core/src/test/org/apache/lucene/search/TestCollectorManager.java

@@ -93,12 +93,14 @@ private static DirectoryReader reader(Directory dir) throws IOException {
   private static <C extends Collector> Object collectAll(
       LeafReaderContext ctx, Collection<Integer> values, CollectorManager<C, ?> collectorManager)
       throws IOException {
+    var rnd = nonAssertingRandom(random());
+
     List<C> collectors = new ArrayList<>();
     C collector = collectorManager.newCollector();
     collectors.add(collector);
     LeafCollector leafCollector = collector.getLeafCollector(ctx);
     for (Integer v : values) {
-      if (random().nextInt(10) == 1) {
+      if (rnd.nextInt(10) == 1) {
         collector = collectorManager.newCollector();
         collectors.add(collector);
         leafCollector = collector.getLeafCollector(ctx);