Speed up java formatting tasks (#15386)

* Enable configuration cache for :help (as a test).

* Enable configuration cache for :showTestsSeed.

* Refactor ModularPathsExtension to enable configuration cache (compileJava works).

* Make the test task support the configuration-cache.

* Speed up google java format even on non-incremental runs by storing our own file state cache.

Author: dweiss

build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/spotless/ApplyGoogleJavaFormatTask.java

@@ -45,29 +45,33 @@ public void formatSources(InputChanges inputChanges) throws IOException {
     WorkQueue workQueue = getWorkQueue();
     List<File> sourceFiles = getIncrementalBatch(inputChanges);
 
-    if (sourceFiles.isEmpty()) {
-      return;
-    }
+    var fileStates = readFileStatesFrom(getFileStateCache());
+    sourceFiles = maybeRefilter(fileStates, sourceFiles);
 
     getLogger()
         .info(
             "Will apply formatting to {} source {} in this run.",
             sourceFiles.size(),
             sourceFiles.size() == 1 ? "file" : "files");
 
-    for (var batch : batchSourceFiles(sourceFiles)) {
-      workQueue.submit(
-          ApplyGoogleJavaFormatAction.class,
-          params -> {
-            params.getTargetFiles().setFrom(batch);
-          });
+    if (!sourceFiles.isEmpty()) {
+      for (var batch : batchSourceFiles(sourceFiles)) {
+        workQueue.submit(
+            ApplyGoogleJavaFormatAction.class,
+            params -> {
+              params.getTargetFiles().setFrom(batch);
+            });
+      }
+
+      workQueue.await();
+
+      updateFileStates(fileStates, sourceFiles);
+      writeFileStates(getFileStateCache(), fileStates);
     }
 
     Path taskOutput = getOutputChangeListFile().get().getAsFile().toPath();
     Files.writeString(
         taskOutput, sourceFiles.stream().map(File::getPath).collect(Collectors.joining("\n")));
-
-    workQueue.await();
   }
 
   public abstract static class ApplyGoogleJavaFormatAction

build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/spotless/CheckGoogleJavaFormatTask.java

@@ -82,53 +82,60 @@ public CheckGoogleJavaFormatTask(ProjectLayout layout) {
 
   @TaskAction
   public void formatSources(InputChanges inputChanges) throws IOException {
-    WorkQueue workQueue = getWorkQueue();
     List<File> sourceFiles = getIncrementalBatch(inputChanges);
-    if (sourceFiles.isEmpty()) {
-      return;
-    }
+
+    var fileStates = readFileStatesFrom(getFileStateCache());
+    sourceFiles = maybeRefilter(fileStates, sourceFiles);
 
     getLogger()
         .info(
             "Will check the formatting of {} source {} in this run.",
             sourceFiles.size(),
             sourceFiles.size() == 1 ? "file" : "files");
 
-    int outputSeq = 0;
-    File tempDir = new File(getTemporaryDir(), "changes");
-    if (Files.exists(tempDir.toPath())) {
-      getFilesystemOperations().delete(spec -> spec.delete(tempDir));
-    }
-    Files.createDirectory(tempDir.toPath());
-
-    for (List<File> batch : batchSourceFiles(sourceFiles)) {
-      final int seq = outputSeq;
-      workQueue.submit(
-          CheckGoogleJavaFormatAction.class,
-          params -> {
-            params.getTargetFiles().setFrom(batch);
-            params.getOutputFile().set(new File(tempDir, "gjf-check-" + seq + ".txt"));
-          });
-      outputSeq++;
-    }
-    workQueue.await();
+    if (!sourceFiles.isEmpty()) {
+      int outputSeq = 0;
+      File tempDir = new File(getTemporaryDir(), "changes");
+      if (Files.exists(tempDir.toPath())) {
+        getFilesystemOperations().delete(spec -> spec.delete(tempDir));
+      }
+      Files.createDirectory(tempDir.toPath());
+
+      WorkQueue workQueue = getWorkQueue();
+      for (List<File> batch : batchSourceFiles(sourceFiles)) {
+        final int seq = outputSeq;
+        workQueue.submit(
+            CheckGoogleJavaFormatAction.class,
+            params -> {
+              params.getTargetFiles().setFrom(batch);
+              params.getOutputFile().set(new File(tempDir, "gjf-check-" + seq + ".txt"));
+            });
+        outputSeq++;
+      }
 
-    // Check if there are any changes.
-    try (Stream<Path> stream = Files.list(tempDir.toPath()).sorted()) {
-      var allChanges = stream.toList();
+      // Wait for all jobs.
+      workQueue.await();
 
-      if (!allChanges.isEmpty()) {
-        var errorMsg =
-            "java file(s) have google-java-format violations (run './gradlew tidy' to fix).";
+      // Check if there are any changes.
+      try (Stream<Path> stream = Files.list(tempDir.toPath()).sorted()) {
+        var allChanges = stream.toList();
 
-        if (getColorizedOutput().getOrElse(false)) {
-          printChangesToColorizedTerminal(allChanges, errorMsg);
-        } else {
-          printChangesToLogger(allChanges, errorMsg);
-        }
+        if (!allChanges.isEmpty()) {
+          var errorMsg =
+              "java file(s) have google-java-format violations (run './gradlew tidy' to fix).";
+
+          if (getColorizedOutput().getOrElse(false)) {
+            printChangesToColorizedTerminal(allChanges, errorMsg);
+          } else {
+            printChangesToLogger(allChanges, errorMsg);
+          }
 
-        throw new GradleException(errorMsg);
+          throw new GradleException(errorMsg);
+        }
       }
+
+      updateFileStates(fileStates, sourceFiles);
+      writeFileStates(getFileStateCache(), fileStates);
     }
 
     Path taskOutput = getOutputChangeListFile().get().getAsFile().toPath();

build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/spotless/FileState.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.gradle.plugins.spotless;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * The "state" of a source file. We'll just follow rsync and assume that a file with the same size
+ * and the same last-update timestamp hasn't changed.
+ */
+record FileState(long size, long lastUpdate) {
+  public static FileState of(Path path) throws IOException {
+    return new FileState(Files.size(path), Files.getLastModifiedTime(path).toMillis());
+  }
+}

build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/spotless/GoogleJavaFormatPlugin.java

@@ -67,16 +67,17 @@ public void apply(Project project) {
     tasks.named("tidy", tidy -> tidy.dependsOn(applyTask));
     tasks.named("check", check -> check.dependsOn(checkTask));
 
+    var fileStates = project.getLayout().getBuildDirectory().file("gjf-file-states.json");
     for (var t : List.of(applyTask, checkTask)) {
       t.configure(
           task -> {
             task.getBatchSize().set(batchSizeOption);
             task.dependsOn(":" + CheckEnvironmentPlugin.TASK_CHECK_JDK_INTERNALS_EXPOSED_TO_GRADLE);
+            task.getFileStateCache().set(fileStates);
           });
     }
 
     // Configure details depending on the project.
-
     for (var t : List.of(applyTask, checkTask)) {
       t.configure(
           task -> {

build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/spotless/ParentGoogleJavaFormatTask.java

@@ -22,8 +22,18 @@
 import com.google.googlejavaformat.java.ImportOrderer;
 import com.google.googlejavaformat.java.JavaFormatterOptions;
 import com.google.googlejavaformat.java.RemoveUnusedImports;
+import groovy.json.JsonOutput;
+import groovy.json.JsonSlurper;
 import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 import javax.inject.Inject;
 import org.gradle.api.DefaultTask;
@@ -63,10 +73,18 @@ abstract class ParentGoogleJavaFormatTask extends DefaultTask {
   @Inject
   protected abstract WorkerExecutor getWorkerExecutor();
 
+  @Internal
+  protected abstract RegularFileProperty getFileStateCache();
+
+  private Function<File, String> fileToKey;
+
   public ParentGoogleJavaFormatTask(ProjectLayout layout, String gjfTask) {
     getOutputChangeListFile()
         .convention(layout.getBuildDirectory().file("gjf-" + gjfTask + ".txt"));
     getBatchSize().convention(1);
+
+    var projectPath = layout.getProjectDirectory().getAsFile().toPath();
+    fileToKey = file -> projectPath.relativize(file.toPath()).toString();
   }
 
   protected Iterable<List<File>> batchSourceFiles(List<File> sourceFiles) {
@@ -120,10 +138,78 @@ protected static String applyFormatter(Formatter formatter, String input)
     return input;
   }
 
+  protected void writeFileStates(
+      RegularFileProperty pathProvider, TreeMap<String, FileState> fileStates) throws IOException {
+    if (pathProvider.isPresent()) {
+      Files.writeString(
+          pathProvider.get().getAsFile().toPath(),
+          JsonOutput.prettyPrint(JsonOutput.toJson(fileStates)));
+    }
+  }
+
+  protected final TreeMap<String, FileState> readFileStatesFrom(RegularFileProperty pathProvider) {
+    var checksums = new TreeMap<String, FileState>();
+    if (pathProvider.isPresent()) {
+      Path path = pathProvider.get().getAsFile().toPath();
+      if (Files.exists(path)) {
+        try {
+          @SuppressWarnings("unchecked")
+          var saved = (Map<String, Map<String, Object>>) new JsonSlurper().parse(path);
+          var restored =
+              saved.entrySet().stream()
+                  .collect(
+                      Collectors.toMap(
+                          Map.Entry::getKey,
+                          e ->
+                              new FileState(
+                                  ((Number) e.getValue().get("size")).longValue(),
+                                  ((Number) e.getValue().get("lastUpdate")).longValue())));
+          checksums.putAll(restored);
+        } catch (Exception _) {
+          getLogger().warn("Could not deserialize changes file: {}, ignoring it.", path);
+        }
+      }
+    }
+
+    return checksums;
+  }
+
+  protected final List<File> maybeRefilter(
+      TreeMap<String, FileState> fileStates, List<File> sourceFiles) throws IOException {
+    if (!fileStates.isEmpty()) {
+      List<File> filteredSourceFiles = new ArrayList<>(sourceFiles);
+      for (var it = filteredSourceFiles.iterator(); it.hasNext(); ) {
+        var file = it.next();
+        var p = file.toPath();
+        var state = fileStates.get(fileToKey.apply(file));
+        if (state != null && FileState.of(p).equals(state)) {
+          it.remove();
+        }
+      }
+
+      if (sourceFiles.size() != filteredSourceFiles.size()) {
+        getLogger()
+            .info(
+                "Reduced the number of files to check from {} to {} based on stored file states.",
+                sourceFiles.size(),
+                filteredSourceFiles.size());
+      }
+
+      sourceFiles = filteredSourceFiles;
+    }
+    return sourceFiles;
+  }
+
+  protected final void updateFileStates(
+      TreeMap<String, FileState> fileStates, List<File> sourceFiles) throws IOException {
+    for (var file : sourceFiles) {
+      fileStates.put(fileToKey.apply(file), FileState.of(file.toPath()));
+    }
+  }
+
   @Internal
   protected WorkQueue getWorkQueue() {
-    // TODO: maybe fork a separate jvm so that we can pass open-module settings there and fine-tune
-    // the jvm for the task?
+    // Keep workers within the same JVM.
     return getWorkerExecutor().noIsolation();
   }
 }