Merge branch 'apache:main' into fst-streamline

Author: dungba88

gradle/testing/randomization.gradle

@@ -67,7 +67,7 @@ allprojects {
           // seed, repetition and amplification.
           [propName: 'tests.seed', value: { -> rootSeed }, description: "Sets the master randomization seed."],
           [propName: 'tests.iters', value: null, description: "Duplicate (re-run) each test case N times."],
-          [propName: 'tests.multiplier', value: 1, description: "Value multiplier for randomized tests."],
+          [propName: 'tests.multiplier', value: null, description: "Value multiplier for randomized tests."],
           [propName: 'tests.maxfailures', value: null, description: "Skip tests after a given number of failures."],
           [propName: 'tests.timeoutSuite', value: null, description: "Timeout (in millis) for an entire suite."],
           [propName: 'tests.failfast', value: "false", description: "Stop the build early on failure.", buildOnly: true],

lucene/CHANGES.txt

@@ -200,8 +200,8 @@ Improvements
 * GITHUB#12677: Better detect vector module in non-default setups (e.g., custom module layers).
   (Uwe Schindler)
 
-* GITHUB#12634, GITHUB#12632, GITHUB#12680, GITHUB#12681, GITHUB#12731: Speed up Panama vector support
-  and test improvements. (Uwe Schindler, Robert Muir)
+* GITHUB#12634, GITHUB#12632, GITHUB#12680, GITHUB#12681, GITHUB#12731, GITHUB#12737: Speed up
+  Panama vector support and test improvements. (Uwe Schindler, Robert Muir)
 
 * GITHUB#12586: Remove over-counting of deleted terms. (Guo Feng)
 
@@ -210,6 +210,9 @@ Improvements
 
 * GITHUB#12689: TaskExecutor to cancel all tasks on exception to avoid needless computation. (Luca Cavanna)
 
+* GITHUB#12754: Refactor lookup of Hotspot VM options and do not initialize constants with NULL
+  if SecurityManager prevents access.  (Uwe Schindler)
+
 Optimizations
 ---------------------
 * GITHUB#12183: Make TermStates#build concurrent. (Shubham Chaudhary)
@@ -264,6 +267,8 @@ Changes in runtime behavior
 * GITHUB#12569: Prevent concurrent tasks from parallelizing execution further which could cause deadlock
   (Luca Cavanna)
 
+* GITHUB#12765: Disable vectorization on VMs that are not Hotspot-based.  (Uwe Schindler, Robert Muir)
+
 Bug Fixes
 ---------------------
 
@@ -285,6 +290,9 @@ Bug Fixes
 Build
 ---------------------
 
+* GITHUB#12752: tests.multiplier could be omitted in test failure reproduce lines (esp. in
+  nightly mode). (Dawid Weiss)
+
 * GITHUB#12742: JavaCompile tasks may be in up-to-date state when modular dependencies have changed 
   leading to odd runtime errors (Chris Hostetter, Dawid Weiss)
 

lucene/analysis/common/src/tools/java/org/apache/lucene/analysis/standard/GenerateJflexTLDMacros.java

@@ -21,6 +21,7 @@
 import java.io.InputStreamReader;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
+import java.net.URI;
 import java.net.URL;
 import java.net.URLConnection;
 import java.nio.charset.StandardCharsets;
@@ -111,7 +112,7 @@ public static void main(String... args) throws Exception {
 
   public GenerateJflexTLDMacros(String tldFileURL, String jflexFile, String tldListFile)
       throws Exception {
-    this.tldFileURL = new URL(tldFileURL);
+    this.tldFileURL = URI.create(tldFileURL).toURL();
     this.jflexMacroFile = Paths.get(jflexFile);
     this.tldListFile = Paths.get(tldListFile);
   }

lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java

@@ -27,6 +27,7 @@
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
+import java.net.URI;
 import java.net.URL;
 import java.net.URLConnection;
 import java.nio.charset.StandardCharsets;
@@ -155,19 +156,19 @@ private static void expandDataFileRules(Path file) throws IOException {
   }
 
   private static void getNFKCDataFilesFromIcuProject(String releaseTag) throws IOException {
-    URL icuTagsURL = new URL(ICU_GIT_TAG_URL + "/");
-    URL icuReleaseTagURL = new URL(icuTagsURL, releaseTag + "/");
-    URL norm2url = new URL(icuReleaseTagURL, ICU_DATA_NORM2_PATH + "/");
+    URI icuTagsURI = URI.create(ICU_GIT_TAG_URL + "/");
+    URI icuReleaseTagURI = icuTagsURI.resolve(releaseTag + "/");
+    URI norm2uri = icuReleaseTagURI.resolve(ICU_DATA_NORM2_PATH + "/");
 
     System.err.print("Downloading " + NFKC_TXT + " ... ");
-    download(new URL(norm2url, NFKC_TXT), NFKC_TXT);
+    download(norm2uri.resolve(NFKC_TXT), NFKC_TXT);
     System.err.println("done.");
     System.err.print("Downloading " + NFKC_CF_TXT + " ... ");
-    download(new URL(norm2url, NFKC_CF_TXT), NFKC_CF_TXT);
+    download(norm2uri.resolve(NFKC_CF_TXT), NFKC_CF_TXT);
     System.err.println("done.");
 
     System.err.print("Downloading " + NFKC_CF_TXT + " and making diacritic rules one-way ... ");
-    URLConnection connection = openConnection(new URL(norm2url, NFC_TXT));
+    URLConnection connection = openConnection(norm2uri.resolve(NFC_TXT).toURL());
     try (BufferedReader reader =
             new BufferedReader(
                 new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8));
@@ -210,8 +211,8 @@ private static void getNFKCDataFilesFromIcuProject(String releaseTag) throws IOE
     System.err.println("done.");
   }
 
-  private static void download(URL url, String outputFile) throws IOException {
-    final URLConnection connection = openConnection(url);
+  private static void download(URI uri, String outputFile) throws IOException {
+    final URLConnection connection = openConnection(uri.toURL());
     try (InputStream inputStream = connection.getInputStream();
         OutputStream outputStream = Files.newOutputStream(Path.of(outputFile))) {
       inputStream.transferTo(outputStream);

lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.java

@@ -17,72 +17,46 @@
 
 package org.apache.lucene.internal.vectorization;
 
+import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.SuppressForbidden;
+
 final class DefaultVectorUtilSupport implements VectorUtilSupport {
 
   DefaultVectorUtilSupport() {}
 
+  // the way FMA should work! if available use it, otherwise fall back to mul/add
+  @SuppressForbidden(reason = "Uses FMA only where fast and carefully contained")
+  private static float fma(float a, float b, float c) {
+    if (Constants.HAS_FAST_SCALAR_FMA) {
+      return Math.fma(a, b, c);
+    } else {
+      return a * b + c;
+    }
+  }
+
   @Override
   public float dotProduct(float[] a, float[] b) {
     float res = 0f;
-    /*
-     * If length of vector is larger than 8, we use unrolled dot product to accelerate the
-     * calculation.
-     */
-    int i;
-    for (i = 0; i < a.length % 8; i++) {
-      res += b[i] * a[i];
-    }
-    if (a.length < 8) {
-      return res;
-    }
-    for (; i + 31 < a.length; i += 32) {
-      res +=
-          b[i + 0] * a[i + 0]
-              + b[i + 1] * a[i + 1]
-              + b[i + 2] * a[i + 2]
-              + b[i + 3] * a[i + 3]
-              + b[i + 4] * a[i + 4]
-              + b[i + 5] * a[i + 5]
-              + b[i + 6] * a[i + 6]
-              + b[i + 7] * a[i + 7];
-      res +=
-          b[i + 8] * a[i + 8]
-              + b[i + 9] * a[i + 9]
-              + b[i + 10] * a[i + 10]
-              + b[i + 11] * a[i + 11]
-              + b[i + 12] * a[i + 12]
-              + b[i + 13] * a[i + 13]
-              + b[i + 14] * a[i + 14]
-              + b[i + 15] * a[i + 15];
-      res +=
-          b[i + 16] * a[i + 16]
-              + b[i + 17] * a[i + 17]
-              + b[i + 18] * a[i + 18]
-              + b[i + 19] * a[i + 19]
-              + b[i + 20] * a[i + 20]
-              + b[i + 21] * a[i + 21]
-              + b[i + 22] * a[i + 22]
-              + b[i + 23] * a[i + 23];
-      res +=
-          b[i + 24] * a[i + 24]
-              + b[i + 25] * a[i + 25]
-              + b[i + 26] * a[i + 26]
-              + b[i + 27] * a[i + 27]
-              + b[i + 28] * a[i + 28]
-              + b[i + 29] * a[i + 29]
-              + b[i + 30] * a[i + 30]
-              + b[i + 31] * a[i + 31];
+    int i = 0;
+
+    // if the array is big, unroll it
+    if (a.length > 32) {
+      float acc1 = 0;
+      float acc2 = 0;
+      float acc3 = 0;
+      float acc4 = 0;
+      int upperBound = a.length & ~(4 - 1);
+      for (; i < upperBound; i += 4) {
+        acc1 = fma(a[i], b[i], acc1);
+        acc2 = fma(a[i + 1], b[i + 1], acc2);
+        acc3 = fma(a[i + 2], b[i + 2], acc3);
+        acc4 = fma(a[i + 3], b[i + 3], acc4);
+      }
+      res += acc1 + acc2 + acc3 + acc4;
     }
-    for (; i + 7 < a.length; i += 8) {
-      res +=
-          b[i + 0] * a[i + 0]
-              + b[i + 1] * a[i + 1]
-              + b[i + 2] * a[i + 2]
-              + b[i + 3] * a[i + 3]
-              + b[i + 4] * a[i + 4]
-              + b[i + 5] * a[i + 5]
-              + b[i + 6] * a[i + 6]
-              + b[i + 7] * a[i + 7];
+
+    for (; i < a.length; i++) {
+      res = fma(a[i], b[i], res);
     }
     return res;
   }
@@ -92,50 +66,80 @@ public float cosine(float[] a, float[] b) {
     float sum = 0.0f;
     float norm1 = 0.0f;
     float norm2 = 0.0f;
-    int dim = a.length;
+    int i = 0;
 
-    for (int i = 0; i < dim; i++) {
-      float elem1 = a[i];
-      float elem2 = b[i];
-      sum += elem1 * elem2;
-      norm1 += elem1 * elem1;
-      norm2 += elem2 * elem2;
+    // if the array is big, unroll it
+    if (a.length > 32) {
+      float sum1 = 0;
+      float sum2 = 0;
+      float norm1_1 = 0;
+      float norm1_2 = 0;
+      float norm2_1 = 0;
+      float norm2_2 = 0;
+
+      int upperBound = a.length & ~(2 - 1);
+      for (; i < upperBound; i += 2) {
+        // one
+        sum1 = fma(a[i], b[i], sum1);
+        norm1_1 = fma(a[i], a[i], norm1_1);
+        norm2_1 = fma(b[i], b[i], norm2_1);
+
+        // two
+        sum2 = fma(a[i + 1], b[i + 1], sum2);
+        norm1_2 = fma(a[i + 1], a[i + 1], norm1_2);
+        norm2_2 = fma(b[i + 1], b[i + 1], norm2_2);
+      }
+      sum += sum1 + sum2;
+      norm1 += norm1_1 + norm1_2;
+      norm2 += norm2_1 + norm2_2;
+    }
+
+    for (; i < a.length; i++) {
+      sum = fma(a[i], b[i], sum);
+      norm1 = fma(a[i], a[i], norm1);
+      norm2 = fma(b[i], b[i], norm2);
     }
     return (float) (sum / Math.sqrt((double) norm1 * (double) norm2));
   }
 
   @Override
   public float squareDistance(float[] a, float[] b) {
-    float squareSum = 0.0f;
-    int dim = a.length;
-    int i;
-    for (i = 0; i + 8 <= dim; i += 8) {
-      squareSum += squareDistanceUnrolled(a, b, i);
+    float res = 0;
+    int i = 0;
+
+    // if the array is big, unroll it
+    if (a.length > 32) {
+      float acc1 = 0;
+      float acc2 = 0;
+      float acc3 = 0;
+      float acc4 = 0;
+
+      int upperBound = a.length & ~(4 - 1);
+      for (; i < upperBound; i += 4) {
+        // one
+        float diff1 = a[i] - b[i];
+        acc1 = fma(diff1, diff1, acc1);
+
+        // two
+        float diff2 = a[i + 1] - b[i + 1];
+        acc2 = fma(diff2, diff2, acc2);
+
+        // three
+        float diff3 = a[i + 2] - b[i + 2];
+        acc3 = fma(diff3, diff3, acc3);
+
+        // four
+        float diff4 = a[i + 3] - b[i + 3];
+        acc4 = fma(diff4, diff4, acc4);
+      }
+      res += acc1 + acc2 + acc3 + acc4;
     }
-    for (; i < dim; i++) {
+
+    for (; i < a.length; i++) {
       float diff = a[i] - b[i];
-      squareSum += diff * diff;
+      res = fma(diff, diff, res);
     }
-    return squareSum;
-  }
-
-  private static float squareDistanceUnrolled(float[] v1, float[] v2, int index) {
-    float diff0 = v1[index + 0] - v2[index + 0];
-    float diff1 = v1[index + 1] - v2[index + 1];
-    float diff2 = v1[index + 2] - v2[index + 2];
-    float diff3 = v1[index + 3] - v2[index + 3];
-    float diff4 = v1[index + 4] - v2[index + 4];
-    float diff5 = v1[index + 5] - v2[index + 5];
-    float diff6 = v1[index + 6] - v2[index + 6];
-    float diff7 = v1[index + 7] - v2[index + 7];
-    return diff0 * diff0
-        + diff1 * diff1
-        + diff2 * diff2
-        + diff3 * diff3
-        + diff4 * diff4
-        + diff5 * diff5
-        + diff6 * diff6
-        + diff7 * diff7;
+    return res;
   }
 
   @Override

lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java

@@ -21,8 +21,6 @@
 import java.lang.StackWalker.StackFrame;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
 import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
@@ -31,7 +29,7 @@
 import java.util.function.Predicate;
 import java.util.logging.Logger;
 import java.util.stream.Stream;
-import org.apache.lucene.util.SuppressForbidden;
+import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.VectorUtil;
 
 /**
@@ -113,6 +111,12 @@ static VectorizationProvider lookup(boolean testMode) {
                 + Locale.getDefault());
         return new DefaultVectorizationProvider();
       }
+      // only use vector module with Hotspot VM
+      if (!Constants.IS_HOTSPOT_VM) {
+        LOG.warning(
+            "Java runtime is not using Hotspot VM; Java vector incubator API can't be enabled.");
+        return new DefaultVectorizationProvider();
+      }
       // is the incubator module present and readable (JVM providers may to exclude them or it is
       // build with jlink)
       final var vectorMod = lookupVectorModule();
@@ -129,7 +133,7 @@ static VectorizationProvider lookup(boolean testMode) {
               "Vector bitsize and/or integer vectors enforcement; using default vectorization provider outside of testMode");
           return new DefaultVectorizationProvider();
         }
-        if (isClientVM()) {
+        if (Constants.IS_CLIENT_VM) {
           LOG.warning("C2 compiler is disabled; Java vector incubator API can't be enabled");
           return new DefaultVectorizationProvider();
         }
@@ -188,23 +192,6 @@ private static boolean isAffectedByJDK8301190() {
         && !Objects.equals("I", "i".toUpperCase(Locale.getDefault()));
   }
 
-  @SuppressWarnings("removal")
-  @SuppressForbidden(reason = "security manager")
-  private static boolean isClientVM() {
-    try {
-      final PrivilegedAction<Boolean> action =
-          () -> System.getProperty("java.vm.info", "").contains("emulated-client");
-      return AccessController.doPrivileged(action);
-    } catch (
-        @SuppressWarnings("unused")
-        SecurityException e) {
-      LOG.warning(
-          "SecurityManager denies permission to 'java.vm.info' system property, so state of C2 compiler can't be detected. "
-              + "In case of performance issues allow access to this property.");
-      return false;
-    }
-  }
-
   // add all possible callers here as FQCN:
   private static final Set<String> VALID_CALLERS = Set.of("org.apache.lucene.util.VectorUtil");