Merge branch 'main' into compiler-metadata

Author: dungba88

dev-tools/README.txt

@@ -5,8 +5,8 @@ as to the usefulness of the tools.
 
 Description of dev-tools/ contents:
 
-./size-estimator-lucene-solr.xls -- Spreadsheet for estimating memory and disk usage in Lucene/Solr
+./aws-jmh/       -- Scripts for running microbenchmarks across different ec2 instance types.
+./doap/          -- Lucene project descriptors in DOAP RDF format.
 ./missing-doclet -- JavaDoc validation doclet subproject
-./doap/       -- Lucene and Solr project descriptors in DOAP RDF format.
-./scripts/    -- Odds and ends for building releases, etc.
-./test-patch/ -- Scripts for automatically validating patches
+./scripts/       -- Odds and ends for building releases, etc.
+./test-patch/    -- Scripts for automatically validating patches

dev-tools/doap/lucene.rdf

@@ -67,6 +67,20 @@
     </maintainer>
 
     <!-- NOTE: please insert releases in numeric order, NOT chronologically. -->
+    <release>
+       <Version>
+         <name>lucene-9.9.1</name>
+         <created>2023-12-16</created>
+         <revision>9.9.1</revision>
+       </Version>
+    </release>
+    <release>
+       <Version>
+         <name>lucene-9.9.0</name>
+         <created>2023-12-04</created>
+         <revision>9.9.0</revision>
+       </Version>
+    </release>
     <release>
        <Version>
          <name>lucene-9.8.0</name>

gradle/generation/extract-jdk-apis.gradle

@@ -62,7 +62,8 @@ configure(project(":lucene:core")) {
       classpath = configurations.apiextractor
       mainClass = file("${resources}/ExtractJdkApis.java") as String
       systemProperties = [
-        'user.timezone': 'UTC'
+        'user.timezone': 'UTC',
+        'file.encoding': 'UTF-8',
       ]
       args = [
         jdkVersion,

gradle/generation/extract-jdk-apis/ExtractJdkApis.java

@@ -189,7 +189,7 @@ public void visitInnerClass(String name, String outerName, String innerName, int
     }
 
     @Override
-    public void visitPermittedSubclass​(String c) {
+    public void visitPermittedSubclass(String c) {
     }
 
   }

gradle/help.gradle

@@ -46,7 +46,7 @@ configure(rootProject) {
   help {
     doLast {
       println ""
-      println "This is an experimental Lucene/Solr gradle build. See some"
+      println "This is Lucene's gradle build. See some"
       println "guidelines, ant-equivalent commands etc. under help/*; or type:"
       println ""
       helpFiles.each { section, path, sectionInfo ->

gradle/testing/randomization.gradle

@@ -144,7 +144,6 @@ allprojects {
          "tests.leaveTemporary",
          "tests.leavetemporary",
          "tests.leavetmpdir",
-         "solr.test.leavetmpdir",
       ].find { prop ->
         def v = Boolean.parseBoolean(propertyOrDefault(prop, "false"))
         if (v) {

gradle/testing/randomization/policies/tests.policy

@@ -77,11 +77,6 @@ grant {
   // used by nested tests? (e.g. TestLeaveFilesIfTestFails). TODO: look into this
   permission java.util.PropertyPermission "tests.runnested", "write";
 
-  // solr properties. TODO: move these out to SolrTestCase
-  permission java.util.PropertyPermission "solr.data.dir", "write";
-  permission java.util.PropertyPermission "solr.solr.home", "write";
-  permission java.util.PropertyPermission "solr.directoryFactory", "write";
-
   // allows LuceneTestCase#runWithRestrictedPermissions to execute with lower (or no) permission
   permission java.security.SecurityPermission "createAccessControlContext";
 

gradle/validation/error-prone.gradle

@@ -244,7 +244,7 @@ allprojects { prj ->
             // '-Xep:TryFailThrowable:OFF',
             '-Xep:TypeParameterQualifier:ERROR',
             '-Xep:UnicodeDirectionalityCharacters:ERROR',
-            // '-Xep:UnicodeInCode:OFF', // noisy (spatial3d)
+            '-Xep:UnicodeInCode:ERROR',
             '-Xep:UnnecessaryCheckNotNull:ERROR',
             '-Xep:UnnecessaryTypeArgument:ERROR',
             '-Xep:UnsafeWildcard:ERROR',

gradle/validation/validate-source-patterns.gradle

@@ -15,6 +15,10 @@
  * limitations under the License.
  */
 
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
+
 import org.apache.rat.Defaults
 import org.apache.rat.document.impl.FileDocument
 import org.apache.rat.api.MetaData
@@ -144,8 +148,8 @@ class ValidateSourcePatternsTask extends DefaultTask {
       (~$/\$$Id\b/$) : 'svn keyword',
       (~$/\$$Header\b/$) : 'svn keyword',
       (~$/\$$Source\b/$) : 'svn keyword',
-      (~$/^\uFEFF/$) : 'UTF-8 byte order mark',
-      (~$/import java\.lang\.\w+;/$) : 'java.lang import is unnecessary'
+      (~$/[\u200B\uFEFF]/$) : 'UTF-8 byte order mark or other zero-width codepoints',
+      (~$/import java\.lang\.\w+;/$) : 'java.lang import is unnecessary',
     ]
 
     // Python and others merrily use var declarations, this is a problem _only_ in Java at least for 8x where we're forbidding var declarations
@@ -198,15 +202,29 @@ class ValidateSourcePatternsTask extends DefaultTask {
     ProgressLogger progress = progressLoggerFactory.newOperation(this.class)
     progress.start(this.name, this.name)
 
+    def validatingDecoder = StandardCharsets.UTF_8.newDecoder()
+      .onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT)
+
     sourceFiles.each { f ->
       try {
         progress.progress("Scanning ${f.name}")
         logger.debug('Scanning source file: {}', f);
 
-        def text = f.getText('UTF-8');
+        String text
+        try {
+          validatingDecoder.reset()
+          text = f.withInputStream {
+            in -> new InputStreamReader(in, validatingDecoder).getText()
+          }
+        } catch (CharacterCodingException e) {
+          reportViolation(f, "incorrect UTF-8 encoding [${e}]")
+          return // we can't proceed for this file
+        }
+
         invalidPatterns.each { pattern, name ->
-          if (pattern.matcher(text).find()) {
-            reportViolation(f, name);
+          def matcher = pattern.matcher(text);
+          if (matcher.find()) {
+            reportViolation(f, String.format(Locale.ROOT, '%s [start=%d, end=%d]', name, matcher.start(), matcher.end()));
           }
         }
         def javadocsMatcher = javadocsPattern.matcher(text);
@@ -230,9 +248,10 @@ class ValidateSourcePatternsTask extends DefaultTask {
           }
           checkLicenseHeaderPrecedes(f, 'package', packagePattern, javaCommentPattern, text, ratDocument);
 
-          invalidJavaOnlyPatterns.each { pattern,name ->
-            if (pattern.matcher(text).find()) {
-              reportViolation(f, name);
+          invalidJavaOnlyPatterns.each { pattern, name ->
+            def matcher = pattern.matcher(text);
+            if (matcher.find()) {
+              reportViolation(f, String.format(Locale.ROOT, '%s [start=%d, end=%d]', name, matcher.start(), matcher.end()));
             }
           }
         }

help/tests.txt

@@ -139,7 +139,7 @@ Larger heap size
 By default tests run with a 512 MB max heap.  But some tests (monster/nightly)
 need more heap.  Use "-Dtests.heapsize" for this:
 
-  gradlew -p lucene/core test --tests "Test2BFST" -Dtest.heapsize=32g
+  gradlew -p lucene/core test --tests "Test2BFST" -Dtests.heapsize=32g
 
 
 Run GUI tests headlessly with Xvfb (Linux only)