improve 'javadoc-style-license-header' and 'license-after-package' (#14849)

these two checks are currently implemented with regex rules looking for
comments, then calling into rat API to check if it is a license. It
means parsing every java file with rat just for this purpose, and the
regexes are not perfect.

The new rules are very simple but find some inconsistencies that weren't
found before: they aren't stricter, the previous code just misses some.

We now only call into rat here for xml files, separately we should
decide if that is worth it. If not, we can reduce the gradle code
further and avoid challenges with Rat API on upgrades.

To me, checking this for the 29 XML files is not worth the hassle of
installing additional C library / configuring `customLanguages` to check
the xml with this tool, I add xml parser in my own personal config but I
think its too much of a hassle for anyone else / CI.

* strengthen rule with wildcard node

This allows stuff in between such as import statements, and finds more
inconsistencies: fix them. The TokenManager associated with the
StandardSyntaxParser also has issues here, it has a package statement
and import statement before the license.

* add tests for additional cases now found and fixed

* javadoc-license: add autofix

* Correct license location in StandardSyntaxParser.

* This adds ASL headers to antlr-generated code and removes antlr generated sources from generated family in rat checks.

---------

Co-authored-by: Dawid Weiss <[email protected]>

Author: rmuir

build-tools/build-infra/src/main/groovy/lucene.regenerate.antlr.gradle

@@ -69,7 +69,27 @@ configure(project(":lucene:expressions")) {
       generatedFiles.each { file ->
         modifyFile(file, { text ->
           text = text.replaceAll("public ((interface|class) Javascript\\w+)", "\$1")
-          text = text.replaceAll("// Generated from .*", "// ANTLR GENERATED CODE: DO NOT EDIT")
+          text = text.replaceAll("// Generated from .*",
+              """
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// ANTLR GENERATED CODE: DO NOT EDIT.
+""")
           return text
         })
       }

build-tools/build-infra/src/main/groovy/lucene.validation.rat-sources.gradle

@@ -217,8 +217,6 @@ class RatTask extends DefaultTask {
         pattern(substring: "Produced by GNUPLOT")
         // snowball stemmers generated by snowball compiler
         pattern(substring: "Generated by Snowball")
-        // parsers generated by antlr
-        pattern(substring: "ANTLR GENERATED CODE")
       }
 
       approvedLicense(familyName: "Apache")

build-tools/build-infra/src/main/groovy/lucene.validation.source-patterns.gradle

@@ -147,11 +147,8 @@ class ValidateSourcePatternsTask extends DefaultTask {
       violations.add(msg)
     }
 
-    def javadocsPattern = ~$/(?sm)^\Q/**\E(.*?)\Q*/\E/$;
-    def javaCommentPattern = ~$/(?sm)^\Q/*\E(.*?)\Q*/\E/$;
     def xmlCommentPattern = ~$/(?sm)\Q<!--\E(.*?)\Q-->\E/$;
     def lineSplitter = ~$/[\r\n]+/$;
-    def packagePattern = ~$/(?m)^\s*package\s+org\.apache.*;/$;
     def xmlTagPattern = ~$/(?m)\s*<[a-zA-Z].*/$;
     def validSPINameJavadocTag = ~$/(?s)\s*\*\s*@lucene\.spi\s+\{@value #NAME\}/$;
 
@@ -209,14 +206,6 @@ class ValidateSourcePatternsTask extends DefaultTask {
             reportViolation(f, String.format(Locale.ROOT, '%s [start=%d, end=%d]', name, matcher.start(), matcher.end()));
           }
         }
-        def javadocsMatcher = javadocsPattern.matcher(text);
-        def ratDocument = new FileDocument(f);
-        while (javadocsMatcher.find()) {
-          if (isLicense(javadocsMatcher, ratDocument)) {
-            reportViolation(f, String.format(Locale.ENGLISH, 'javadoc-style license header [%s]',
-                ratDocument.getMetaData().value(MetaData.RAT_URL_LICENSE_FAMILY_NAME)));
-          }
-        }
         if (f.name.endsWith('.java')) {
           // make sure that SPI names of all tokenizers/charfilters/tokenfilters are documented
           if (!f.name.contains("Test") && !f.name.contains("Mock") && !f.name.contains("Fake") && !text.contains("abstract class") &&
@@ -228,9 +217,9 @@ class ValidateSourcePatternsTask extends DefaultTask {
               reportViolation(f, 'invalid spi name documentation')
             }
           }
-          checkLicenseHeaderPrecedes(f, 'package', packagePattern, javaCommentPattern, text, ratDocument);
         }
         if (f.name.endsWith('.xml')) {
+          def ratDocument = new FileDocument(f);
           checkLicenseHeaderPrecedes(f, '<tag>', xmlTagPattern, xmlCommentPattern, text, ratDocument);
         }
       } catch (e) {

gradle/validation/ast-grep/rules/java-patterns.yml

@@ -21,3 +21,33 @@ rule:
 severity: error
 message: illegal use of `var` keyword with generic instance creation
 note: add explicit typing on the RHS when using `var`
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/ast-grep/ast-grep/refs/heads/main/schemas/java_rule.json
+id: javadoc-style-license-header
+language: java
+rule:
+  matches: java-license
+  regex: "^/[*][*]"
+  pattern: $TEXT
+# remove extraneous stars
+transform:
+  NEWTEXT:
+    replace:
+      replace: "^/[*]+"
+      by: "/*"
+      source: $TEXT
+fix: $NEWTEXT
+severity: error
+message: license should be a non-javadoc block comment
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/ast-grep/ast-grep/refs/heads/main/schemas/java_rule.json
+id: license-after-package
+language: java
+rule:
+  kind: package_declaration
+  precedes:
+    matches: java-license
+    # allow anything in between (e.g. import statements)
+    stopBy: end
+severity: error
+message: license should be before `package` declaration

gradle/validation/ast-grep/sgconfig.yml

@@ -1,5 +1,10 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/ast-grep/ast-grep/refs/heads/main/schemas/project.json
 ruleDirs:
   - ./rules
 
 testConfigs:
   - testDir: ./tests
+
+utilDirs:
+  - ./utils

gradle/validation/ast-grep/tests/java-patterns.yml

@@ -13,3 +13,31 @@ invalid:
 valid:
   - HashSet<String> x = new HashSet<>();
   - var list = new ArrayList<String>();
+---
+id: javadoc-style-license-header
+invalid:
+  - |
+    /** BLAH BLAH Unless required by applicable law BLAH BLAH */
+valid:
+  - |
+    /* BLAH BLAH Unless required by applicable law BLAH BLAH */
+  - |
+    /** something else */
+---
+id: license-after-package
+invalid:
+  - |
+    package foo;
+    /* BLAH BLAH Unless required by applicable law BLAH BLAH */
+  - |
+    package foo;
+    import bar;
+    /* BLAH BLAH Unless required by applicable law BLAH BLAH */
+  - |
+    /* BLAH BLAH Unless required by applicable law BLAH BLAH */
+    package foo;
+    /* BLAH BLAH Unless required by applicable law BLAH BLAH */
+valid:
+  - |
+    /* BLAH BLAH Unless required by applicable law BLAH BLAH */
+    package foo;

gradle/validation/ast-grep/utils/java-license.yml

@@ -0,0 +1,14 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/ast-grep/ast-grep/refs/heads/main/schemas/java_rule.json
+id: java-license
+language: java
+rule:
+  # detect a license for non-legal purposes (e.g. style)
+  # make this more sophisticated as needed
+  kind: block_comment
+  # must be top-level
+  inside:
+    kind: program
+  any:
+    - regex: Unless required by applicable law
+    - regex: FITNESS FOR A PARTICULAR PURPOSE

lucene/analysis/common/src/java/org/apache/lucene/analysis/de/GermanLightStemmer.java

@@ -14,7 +14,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.lucene.analysis.de;
 
 /*
  * This algorithm is updated based on code located at:
@@ -51,6 +50,8 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+package org.apache.lucene.analysis.de;
+
 /**
  * Light Stemmer for German.
  *

lucene/analysis/common/src/java/org/apache/lucene/analysis/de/GermanMinimalStemmer.java

@@ -14,7 +14,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.lucene.analysis.de;
 
 /*
  * This algorithm is updated based on code located at:
@@ -51,6 +50,8 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+package org.apache.lucene.analysis.de;
+
 /**
  * Minimal Stemmer for German.
  *

lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishLightStemmer.java

@@ -14,7 +14,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.lucene.analysis.es;
 
 /*
  * This algorithm is updated based on code located at:
@@ -51,6 +50,8 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+package org.apache.lucene.analysis.es;
+
 /**
  * Light Stemmer for Spanish
  *