Add rewrite support for RemoveUnusedImports

.buildkite/pipelines/pull-request/part-7-arm.yml

@@ -0,0 +1,11 @@
+steps:
+  - label: part-7
+    command: |
+      .buildkite/scripts/rewrite.sh
+      .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart7
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-7-fips.yml

@@ -0,0 +1,11 @@
+steps:
+  - label: part-7
+    command: |
+      .buildkite/scripts/rewrite.sh
+      .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart7
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-7-windows.yml

@@ -0,0 +1,11 @@
+steps:
+  - label: part-7
+    command: |
+      .buildkite/scripts/rewrite.sh
+      .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart7
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-7.yml

@@ -0,0 +1,11 @@
+steps:
+  - label: part-7
+    command: |
+      .buildkite/scripts/rewrite.sh
+      .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart7
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/precommit.yml

@@ -6,7 +6,9 @@ steps:
     command: |
       .buildkite/scripts/spotless.sh
       .ci/scripts/run-gradle.sh -Dignore.tests.seed precommit
-    timeout_in_minutes: 300
+      .buildkite/scripts/rewrite.sh
+      .ci/scripts/run-gradle.sh -Dignore.tests.seed precommit
+    timeout_in_minutes: 400
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2404

.buildkite/scripts/rewrite.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+if [[ -z "${BUILDKITE_PULL_REQUEST:-}" ]]; then
+  echo "Not a pull request, skipping rewrite"
+  exit 0
+fi
+
+if ! git diff --exit-code; then
+  echo "Changes are present before running rewrite, not running."
+  git status
+  exit 0
+fi
+
+NEW_COMMIT_MESSAGE="[CI] Auto commit changes from rewrite"
+PREVIOUS_COMMIT_MESSAGE="$(git log -1 --pretty=%B)"
+
+echo "--- Running rewrite"
+.ci/scripts/run-gradle.sh -Dscan.tag.NESTED rewriteRun
+
+if git diff --exit-code; then
+  echo "No changes found after running rewrite. Don't need to auto commit."
+  exit 0
+fi
+
+if [[ "$NEW_COMMIT_MESSAGE" == "$PREVIOUS_COMMIT_MESSAGE" ]]; then
+  echo "Changes found after running rewrite."
+  echo "CI already attempted to commit these changes, but the file(s) seem to have changed again."
+  echo "Please review and fix manually."
+  exit 1
+fi
+
+git config --global user.name elasticsearchmachine
+git config --global user.email '[email protected]'
+
+gh pr checkout "${BUILDKITE_PULL_REQUEST}"
+git add -u .
+git commit -m "$NEW_COMMIT_MESSAGE"
+git push
+
+# After the git push, the new commit will trigger a new build within a few seconds and this build should get cancelled
+# So, let's just sleep to give the build time to cancel itself without an error
+# If it doesn't get cancelled for some reason, then exit with an error, because we don't want this build to be green (we just don't want it to generate an error either)
+sleep 300
+exit 1

BUILDING.md

@@ -80,10 +80,10 @@ For updated or newly added dependencies you need to add an entry to this verific
 
 In case of updating a dependency, ensure to remove the unused entry of the outdated dependency manually from the `verification-metadata.xml` file.
 
-You can also automate the generation of this entry by running your build using the `--write-verification-metadata` commandline option:
-```
-./gradlew --write-verification-metadata sha256 precommit
-```
+- Quick apply via:
+  - `./gradlew --write-verification-metadata sha256 help`
+- Automate the generation of this entry by running your build using the `--write-verification-metadata` commandline option:
+  - `./gradlew --write-verification-metadata sha256 precommit`
 
 The `--write-verification-metadata` Gradle option is generally able to resolve reachable configurations,
 but we use detached configurations for a certain set of plugins and tasks. Therefore, please ensure you run this option with a task that
@@ -92,7 +92,7 @@ uses the changed dependencies. In most cases, `precommit` or `check` are good ca
 We prefer sha256 checksums as md5 and sha1 are not considered safe anymore these days. The generated entry
 will have the `origin` attribute been set to `Generated by Gradle`.
 
-> [!Tip]  
+> [!Tip]
 > A manual confirmation of the Gradle generated checksums is currently not mandatory.
 > If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
 > Please replace the content of the `origin` attribute by `official site` in that case.
@@ -229,7 +229,7 @@ In addition to snapshot builds JitPack supports building Pull Requests. Simply u
 3. Run the Gradle build as needed. Keep in mind the initial resolution might take a bit longer as this needs to be built
 by JitPack in the background before we can resolve the adhoc built dependency.
 
-> [!Note]  
+> [!Note]
 > You should only use that approach locally or on a developer branch for production dependencies as we do
 not want to ship unreleased libraries into our releases.
 
@@ -261,7 +261,7 @@ allprojects {
   ```
 4. Run the Gradle build as needed with `--write-verification-metadata` to ensure the Gradle dependency verification does not fail on your custom dependency.
 
-> [!Note]  
+> [!Note]
 > As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
 flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
 > For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository

CONTRIBUTING.md

@@ -238,16 +238,16 @@ the [Spotless Gradle] plugin. All new projects are automatically formatted,
 while existing projects are gradually being opted-in. The formatting check
 is run automatically via the `precommit` task, but it can be run explicitly with:
 
-    ./gradlew spotlessJavaCheck
+- `./gradlew spotlessJavaCheck rewriteDryRun`
 
 It is usually more useful, and just as fast, to just reformat the project. You
 can do this with:
 
-    ./gradlew spotlessApply
+- `./gradlew spotlessApply rewriteRun`
 
 These tasks can also be run for specific subprojects, e.g.
 
-    ./gradlew server:spotlessJavaCheck
+- `./gradlew server:spotlessJavaCheck server:rewriteDryRun`
 
 Please follow these formatting guidelines:
 

build-conventions/build.gradle

@@ -63,6 +63,10 @@ gradlePlugin {
       id = 'elasticsearch.formatting'
       implementationClass = 'org.elasticsearch.gradle.internal.conventions.precommit.FormattingPrecommitPlugin'
     }
+    rewrite {
+      id = 'elasticsearch.rewrite'
+      implementationClass = 'org.elasticsearch.gradle.internal.conventions.precommit.RewritePrecommitPlugin'
+    }
   }
 }
 
@@ -76,6 +80,7 @@ dependencies {
   api buildLibs.shadow.plugin
   api buildLibs.apache.rat
   api buildLibs.nmcp
+  api buildLibs.rewrite.plugin
   compileOnly buildLibs.checkstyle
   constraints {
     api("org.eclipse.platform:org.eclipse.osgi:3.18.300") {

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/RewritePrecommitPlugin.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.conventions.precommit;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.openrewrite.gradle.RewriteExtension;
+
+import org.openrewrite.gradle.RewritePlugin;
+
+import static java.lang.Boolean.parseBoolean;
+import static java.lang.System.getenv;
+
+/**
+ * This plugin configures formatting for Java source using Spotless
+ * for Gradle. Since the act of formatting existing source can interfere
+ * with developers' workflows, we don't automatically format all code
+ * (yet). Instead, we maintain a list of projects that are excluded from
+ * formatting, until we reach a point where we can comfortably format them
+ * in one go without too much disruption.
+ *
+ * <p>Any new sub-projects must not be added to the exclusions list!
+ *
+ * <p>To perform a reformat, run:
+ *
+ * <pre>    ./gradlew spotlessApply</pre>
+ *
+ * <p>To check the current format, run:
+ *
+ * <pre>    ./gradlew spotlessJavaCheck</pre>
+ *
+ * <p>This is also carried out by the `precommit` task.
+ *
+ * <p>See also the <a href="https://github.com/diffplug/spotless/tree/master/plugin-gradle"
+ * >Spotless project page</a>.
+ */
+public class RewritePrecommitPlugin implements Plugin<Project> {
+
+    private static final boolean IS_NON_CI = parseBoolean(getenv("isCI")) == false;
+    private static final boolean SKIP_FORMATTING = parseBoolean(getenv("skipFormatting"));
+    private static final boolean CODE_CLEANUP = parseBoolean(getenv("codeCleanup"));
+
+    @SuppressWarnings({ "checkstyle:DescendantToken", "checkstyle:LineLength" })
+    @Override
+    public void apply(Project project) {
+        project.getPluginManager().withPlugin("java-base", javaBasePlugin -> {
+            project.getPlugins().apply(PrecommitTaskPlugin.class);
+            project.getPlugins().apply(RewritePlugin.class);
+            project.getRepositories().mavenCentral(); // spotless & rewrite need mavenCentral
+            project.getTasks().named("precommit").configure(precommitTask -> precommitTask.dependsOn( "rewriteDryRun"));
+            project.getTasks().named("check").configure(check -> check.dependsOn("rewriteDryRun"));
+            if (!SKIP_FORMATTING && IS_NON_CI && CODE_CLEANUP) {
+                project.getTasks().named("assemble").configure(check -> check.dependsOn("rewriteRun"));
+            }
+            rewrite(project);
+        });
+    }
+
+    private static void rewrite(Project project) {
+        RewriteExtension rewriteExtension = project.getExtensions().getByType(RewriteExtension.class);
+        rewriteExtension.activeRecipe(
+            "org.openrewrite.java.RemoveUnusedImports"
+            //"org.openrewrite.staticanalysis.RemoveUnusedLocalVariables",
+            //"org.openrewrite.staticanalysis.RemoveUnusedPrivateFields",
+            //"org.openrewrite.staticanalysis.RemoveUnusedPrivateMethods"
+        );
+        rewriteExtension.exclusion("**OpenSearchTestCaseTests.java");
+        rewriteExtension.setExportDatatables(true);
+        rewriteExtension.setFailOnDryRunResults(true);
+    }
+}

build-tools-internal/build.gradle

@@ -18,6 +18,7 @@ plugins {
   id 'elasticsearch.eclipse'
   id 'elasticsearch.versions'
   id 'elasticsearch.formatting'
+  //id 'elasticsearch.rewrite'
 }
 
 group = 'org.elasticsearch.gradle'

build-tools/build.gradle

@@ -22,6 +22,7 @@ plugins {
     id 'elasticsearch.eclipse'
     id 'elasticsearch.versions'
     id 'elasticsearch.formatting'
+    //id 'elasticsearch.rewrite'
 }
 
 description = "The elasticsearch build tools"

build-tools/reaper/build.gradle

@@ -3,6 +3,7 @@ plugins {
   id 'elasticsearch.eclipse'
   id 'elasticsearch.versions'
   id 'elasticsearch.formatting'
+  //id 'elasticsearch.rewrite'
 }
 
 group = "org.elasticsearch.gradle"

build.gradle

@@ -342,7 +342,8 @@ allprojects {
     }
     if (project.path.contains(":libs:cli")) {
       // ensure we resolve p2 dependencies for the spotless eclipse formatter
-      dependsOn "spotlessJavaCheck"
+      dependsOn(spotlessJavaCheck)
+      //dependsOn(spotlessJavaCheck, rewriteDryRun)
     }
 
   }
@@ -442,6 +443,7 @@ allprojects {
   }
 
   apply plugin: 'elasticsearch.formatting'
+  apply plugin: 'elasticsearch.rewrite'
 }
 
 tasks.named("updateDaemonJvm") {

gradle.properties

@@ -1,6 +1,8 @@
 org.gradle.welcome=never
 org.gradle.warning.mode=none
 org.gradle.parallel=true
+# https://github.com/openrewrite/rewrite-gradle-plugin/issues/212
+#org.gradle.workers.max=2
 # We need to declare --add-exports to make spotless working seamlessly with jdk16
 org.gradle.jvmargs=-XX:+HeapDumpOnOutOfMemoryError -Xss2m  --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-opens java.base/java.time=ALL-UNNAMED
 
@@ -16,6 +18,8 @@ org.gradle.java.installations.auto-detect=false
 
 # log some dependency verification info to console
 org.gradle.dependency.verification.console=verbose
+# fixme undo lenient
+org.gradle.dependency.verification=lenient
 
 # allow user to specify toolchain via the RUNTIME_JAVA_HOME environment variable
 org.gradle.java.installations.fromEnv=RUNTIME_JAVA_HOME

gradle/build.versions.toml

@@ -46,6 +46,7 @@ spock-core = { group = "org.spockframework", name="spock-core", version.ref="spo
 spock-junit4 = { group = "org.spockframework", name="spock-junit4", version.ref="spock" }
 spock-platform = { group = "org.spockframework", name="spock-bom", version.ref="spock" }
 spotless-plugin = "com.diffplug.spotless:spotless-plugin-gradle:6.25.0"
+rewrite-plugin = "org.openrewrite.rewrite:org.openrewrite.rewrite.gradle.plugin:7.12.1"
 wiremock = "com.github.tomakehurst:wiremock-jre8-standalone:2.23.2"
 xmlunit-core = "org.xmlunit:xmlunit-core:2.8.2"
 

gradle/verification-metadata.xml

@@ -4221,6 +4221,11 @@
             <sha256 value="33d148db0e11debd0d90677d28242bced907f9c77730000fd597867089039d86" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk7" version="1.9.0">
+         <artifact name="kotlin-stdlib-jdk7-1.9.0.jar">
+            <sha256 value="b7979a7aac94055f0d9f1fd3b47ce5ffe1cb6032a842ba9fbe7186f085289178" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk7" version="1.9.10">
          <artifact name="kotlin-stdlib-jdk7-1.9.10.jar">
             <sha256 value="ac6361bf9ad1ed382c2103d9712c47cdec166232b4903ed596e8876b0681c9b7" origin="Generated by Gradle"/>
@@ -4231,6 +4236,11 @@
             <sha256 value="3db752a30074f06ee6c57984aa6f27da44f4d2bbc7f5442651f6988f1cb2b7d7" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk8" version="1.9.0">
+         <artifact name="kotlin-stdlib-jdk8-1.9.0.jar">
+            <sha256 value="a59fa24fdf1ffb594baecdbf0fd10010f977cea10236d487fe3464977a7377fa" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk8" version="1.9.10">
          <artifact name="kotlin-stdlib-jdk8-1.9.10.jar">
             <sha256 value="a4c74d94d64ce1abe53760fe0389dd941f6fc558d0dab35e47c085a11ec80f28" origin="Generated by Gradle"/>
@@ -4446,6 +4456,16 @@
             <sha256 value="6a5604b5b804e0daca1145df1077609321687734a8b49387e49f10557c186c77" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.openrewrite" name="plugin" version="7.11.0">
+         <artifact name="plugin-7.11.0.jar">
+            <sha256 value="ee345cba36c7c42ebb48b8d49fc83ac639fb552c9e67c2d31d2424f9ab86760e" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.openrewrite" name="plugin" version="7.12.1">
+         <artifact name="plugin-7.12.1.jar">
+            <sha256 value="5ff66a821c4546175c8e44ec5fa1dc5d34aefe6b7dd1b76373df6c5ed7b20d15" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.opensaml" name="opensaml-core" version="4.0.1">
          <artifact name="opensaml-core-4.0.1.jar">
             <sha256 value="27cb366feea67ed897ab01db931403b0413a2de81b930058615e354636c6a14d" origin="Generated by Gradle"/>