diff --git a/.buildkite/pipelines/pull-request/part-7-arm.yml b/.buildkite/pipelines/pull-request/part-7-arm.yml
new file mode 100644
index 0000000000000..ee7215492e8eb
--- /dev/null
+++ b/.buildkite/pipelines/pull-request/part-7-arm.yml
@@ -0,0 +1,11 @@
+steps:
+ - label: part-7
+ command: |
+ .buildkite/scripts/rewrite.sh
+ .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart7
+ timeout_in_minutes: 300
+ agents:
+ provider: gcp
+ image: family/elasticsearch-ubuntu-2404
+ machineType: custom-32-98304
+ buildDirectory: /dev/shm/bk
diff --git a/.buildkite/pipelines/pull-request/part-7-fips.yml b/.buildkite/pipelines/pull-request/part-7-fips.yml
new file mode 100644
index 0000000000000..ee7215492e8eb
--- /dev/null
+++ b/.buildkite/pipelines/pull-request/part-7-fips.yml
@@ -0,0 +1,11 @@
+steps:
+ - label: part-7
+ command: |
+ .buildkite/scripts/rewrite.sh
+ .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart7
+ timeout_in_minutes: 300
+ agents:
+ provider: gcp
+ image: family/elasticsearch-ubuntu-2404
+ machineType: custom-32-98304
+ buildDirectory: /dev/shm/bk
diff --git a/.buildkite/pipelines/pull-request/part-7-windows.yml b/.buildkite/pipelines/pull-request/part-7-windows.yml
new file mode 100644
index 0000000000000..ee7215492e8eb
--- /dev/null
+++ b/.buildkite/pipelines/pull-request/part-7-windows.yml
@@ -0,0 +1,11 @@
+steps:
+ - label: part-7
+ command: |
+ .buildkite/scripts/rewrite.sh
+ .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart7
+ timeout_in_minutes: 300
+ agents:
+ provider: gcp
+ image: family/elasticsearch-ubuntu-2404
+ machineType: custom-32-98304
+ buildDirectory: /dev/shm/bk
diff --git a/.buildkite/pipelines/pull-request/part-7.yml b/.buildkite/pipelines/pull-request/part-7.yml
new file mode 100644
index 0000000000000..ee7215492e8eb
--- /dev/null
+++ b/.buildkite/pipelines/pull-request/part-7.yml
@@ -0,0 +1,11 @@
+steps:
+ - label: part-7
+ command: |
+ .buildkite/scripts/rewrite.sh
+ .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart7
+ timeout_in_minutes: 300
+ agents:
+ provider: gcp
+ image: family/elasticsearch-ubuntu-2404
+ machineType: custom-32-98304
+ buildDirectory: /dev/shm/bk
diff --git a/.buildkite/pipelines/pull-request/precommit.yml b/.buildkite/pipelines/pull-request/precommit.yml
index 7c16ad6311033..f1a38ffac7c37 100644
--- a/.buildkite/pipelines/pull-request/precommit.yml
+++ b/.buildkite/pipelines/pull-request/precommit.yml
@@ -6,7 +6,9 @@ steps:
command: |
.buildkite/scripts/spotless.sh
.ci/scripts/run-gradle.sh -Dignore.tests.seed precommit
- timeout_in_minutes: 300
+ .buildkite/scripts/rewrite.sh
+ .ci/scripts/run-gradle.sh -Dignore.tests.seed precommit
+ timeout_in_minutes: 400
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2404
diff --git a/.buildkite/scripts/rewrite.sh b/.buildkite/scripts/rewrite.sh
new file mode 100755
index 0000000000000..6e466fa426a90
--- /dev/null
+++ b/.buildkite/scripts/rewrite.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+if [[ -z "${BUILDKITE_PULL_REQUEST:-}" ]]; then
+ echo "Not a pull request, skipping rewrite"
+ exit 0
+fi
+
+if ! git diff --exit-code; then
+ echo "Changes are present before running rewrite, not running."
+ git status
+ exit 0
+fi
+
+NEW_COMMIT_MESSAGE="[CI] Auto commit changes from rewrite"
+PREVIOUS_COMMIT_MESSAGE="$(git log -1 --pretty=%B)"
+
+echo "--- Running rewrite"
+.ci/scripts/run-gradle.sh -Dscan.tag.NESTED rewriteRun
+
+if git diff --exit-code; then
+ echo "No changes found after running rewrite. Don't need to auto commit."
+ exit 0
+fi
+
+if [[ "$NEW_COMMIT_MESSAGE" == "$PREVIOUS_COMMIT_MESSAGE" ]]; then
+ echo "Changes found after running rewrite."
+ echo "CI already attempted to commit these changes, but the file(s) seem to have changed again."
+ echo "Please review and fix manually."
+ exit 1
+fi
+
+git config --global user.name elasticsearchmachine
+git config --global user.email 'infra-root+elasticsearchmachine@elastic.co'
+
+gh pr checkout "${BUILDKITE_PULL_REQUEST}"
+git add -u .
+git commit -m "$NEW_COMMIT_MESSAGE"
+git push
+
+# After the git push, the new commit will trigger a new build within a few seconds and this build should get cancelled
+# So, let's just sleep to give the build time to cancel itself without an error
+# If it doesn't get cancelled for some reason, then exit with an error, because we don't want this build to be green (we just don't want it to generate an error either)
+sleep 300
+exit 1
diff --git a/BUILDING.md b/BUILDING.md
index 0dc368140de67..9e0823807ef72 100644
--- a/BUILDING.md
+++ b/BUILDING.md
@@ -80,10 +80,10 @@ For updated or newly added dependencies you need to add an entry to this verific
In case of updating a dependency, ensure to remove the unused entry of the outdated dependency manually from the `verification-metadata.xml` file.
-You can also automate the generation of this entry by running your build using the `--write-verification-metadata` commandline option:
-```
-./gradlew --write-verification-metadata sha256 precommit
-```
+- Quick apply via:
+ - `./gradlew --write-verification-metadata sha256 help`
+- Automate the generation of this entry by running your build using the `--write-verification-metadata` commandline option:
+ - `./gradlew --write-verification-metadata sha256 precommit`
The `--write-verification-metadata` Gradle option is generally able to resolve reachable configurations,
but we use detached configurations for a certain set of plugins and tasks. Therefore, please ensure you run this option with a task that
@@ -92,7 +92,7 @@ uses the changed dependencies. In most cases, `precommit` or `check` are good ca
We prefer sha256 checksums as md5 and sha1 are not considered safe anymore these days. The generated entry
will have the `origin` attribute been set to `Generated by Gradle`.
-> [!Tip]
+> [!Tip]
> A manual confirmation of the Gradle generated checksums is currently not mandatory.
> If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
> Please replace the content of the `origin` attribute by `official site` in that case.
@@ -229,7 +229,7 @@ In addition to snapshot builds JitPack supports building Pull Requests. Simply u
3. Run the Gradle build as needed. Keep in mind the initial resolution might take a bit longer as this needs to be built
by JitPack in the background before we can resolve the adhoc built dependency.
-> [!Note]
+> [!Note]
> You should only use that approach locally or on a developer branch for production dependencies as we do
not want to ship unreleased libraries into our releases.
@@ -261,7 +261,7 @@ allprojects {
```
4. Run the Gradle build as needed with `--write-verification-metadata` to ensure the Gradle dependency verification does not fail on your custom dependency.
-> [!Note]
+> [!Note]
> As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
> For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 326b35af4d196..5df6623c29074 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -238,16 +238,16 @@ the [Spotless Gradle] plugin. All new projects are automatically formatted,
while existing projects are gradually being opted-in. The formatting check
is run automatically via the `precommit` task, but it can be run explicitly with:
- ./gradlew spotlessJavaCheck
+- `./gradlew spotlessJavaCheck rewriteDryRun`
It is usually more useful, and just as fast, to just reformat the project. You
can do this with:
- ./gradlew spotlessApply
+- `./gradlew spotlessApply rewriteRun`
These tasks can also be run for specific subprojects, e.g.
- ./gradlew server:spotlessJavaCheck
+- `./gradlew server:spotlessJavaCheck server:rewriteDryRun`
Please follow these formatting guidelines:
diff --git a/build-conventions/build.gradle b/build-conventions/build.gradle
index 9416c3028e8ee..0a7b0c26c65fa 100644
--- a/build-conventions/build.gradle
+++ b/build-conventions/build.gradle
@@ -63,6 +63,10 @@ gradlePlugin {
id = 'elasticsearch.formatting'
implementationClass = 'org.elasticsearch.gradle.internal.conventions.precommit.FormattingPrecommitPlugin'
}
+ rewrite {
+ id = 'elasticsearch.rewrite'
+ implementationClass = 'org.elasticsearch.gradle.internal.conventions.precommit.RewritePrecommitPlugin'
+ }
}
}
@@ -76,6 +80,7 @@ dependencies {
api buildLibs.shadow.plugin
api buildLibs.apache.rat
api buildLibs.nmcp
+ api buildLibs.rewrite.plugin
compileOnly buildLibs.checkstyle
constraints {
api("org.eclipse.platform:org.eclipse.osgi:3.18.300") {
diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/RewritePrecommitPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/RewritePrecommitPlugin.java
new file mode 100644
index 0000000000000..17bd1bf942705
--- /dev/null
+++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/RewritePrecommitPlugin.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.conventions.precommit;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.openrewrite.gradle.RewriteExtension;
+
+import org.openrewrite.gradle.RewritePlugin;
+
+import static java.lang.Boolean.parseBoolean;
+import static java.lang.System.getenv;
+
+/**
+ * This plugin configures formatting for Java source using Spotless
+ * for Gradle. Since the act of formatting existing source can interfere
+ * with developers' workflows, we don't automatically format all code
+ * (yet). Instead, we maintain a list of projects that are excluded from
+ * formatting, until we reach a point where we can comfortably format them
+ * in one go without too much disruption.
+ *
+ * Any new sub-projects must not be added to the exclusions list!
+ *
+ *
To perform a reformat, run:
+ *
+ *
./gradlew spotlessApply
+ *
+ * To check the current format, run:
+ *
+ *
./gradlew spotlessJavaCheck
+ *
+ * This is also carried out by the `precommit` task.
+ *
+ *
See also the Spotless project page.
+ */
+public class RewritePrecommitPlugin implements Plugin {
+
+ private static final boolean IS_NON_CI = parseBoolean(getenv("isCI")) == false;
+ private static final boolean SKIP_FORMATTING = parseBoolean(getenv("skipFormatting"));
+ private static final boolean CODE_CLEANUP = parseBoolean(getenv("codeCleanup"));
+
+ @SuppressWarnings({ "checkstyle:DescendantToken", "checkstyle:LineLength" })
+ @Override
+ public void apply(Project project) {
+ project.getPluginManager().withPlugin("java-base", javaBasePlugin -> {
+ project.getPlugins().apply(PrecommitTaskPlugin.class);
+ project.getPlugins().apply(RewritePlugin.class);
+ project.getRepositories().mavenCentral(); // spotless & rewrite need mavenCentral
+ project.getTasks().named("precommit").configure(precommitTask -> precommitTask.dependsOn( "rewriteDryRun"));
+ project.getTasks().named("check").configure(check -> check.dependsOn("rewriteDryRun"));
+ if (!SKIP_FORMATTING && IS_NON_CI && CODE_CLEANUP) {
+ project.getTasks().named("assemble").configure(check -> check.dependsOn("rewriteRun"));
+ }
+ rewrite(project);
+ });
+ }
+
+ private static void rewrite(Project project) {
+ RewriteExtension rewriteExtension = project.getExtensions().getByType(RewriteExtension.class);
+ rewriteExtension.activeRecipe(
+ "org.openrewrite.java.RemoveUnusedImports"
+ //"org.openrewrite.staticanalysis.RemoveUnusedLocalVariables",
+ //"org.openrewrite.staticanalysis.RemoveUnusedPrivateFields",
+ //"org.openrewrite.staticanalysis.RemoveUnusedPrivateMethods"
+ );
+ rewriteExtension.exclusion("**OpenSearchTestCaseTests.java");
+ rewriteExtension.setExportDatatables(true);
+ rewriteExtension.setFailOnDryRunResults(true);
+ }
+}
diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle
index 6f5dc5e0ca62c..afcc26f41215d 100644
--- a/build-tools-internal/build.gradle
+++ b/build-tools-internal/build.gradle
@@ -18,6 +18,7 @@ plugins {
id 'elasticsearch.eclipse'
id 'elasticsearch.versions'
id 'elasticsearch.formatting'
+ //id 'elasticsearch.rewrite'
}
group = 'org.elasticsearch.gradle'
diff --git a/build-tools/build.gradle b/build-tools/build.gradle
index 9d9ec139b175b..6422febee21c5 100644
--- a/build-tools/build.gradle
+++ b/build-tools/build.gradle
@@ -22,6 +22,7 @@ plugins {
id 'elasticsearch.eclipse'
id 'elasticsearch.versions'
id 'elasticsearch.formatting'
+ //id 'elasticsearch.rewrite'
}
description = "The elasticsearch build tools"
diff --git a/build-tools/reaper/build.gradle b/build-tools/reaper/build.gradle
index ad50a37d0608f..0f74a12325b6a 100644
--- a/build-tools/reaper/build.gradle
+++ b/build-tools/reaper/build.gradle
@@ -3,6 +3,7 @@ plugins {
id 'elasticsearch.eclipse'
id 'elasticsearch.versions'
id 'elasticsearch.formatting'
+ //id 'elasticsearch.rewrite'
}
group = "org.elasticsearch.gradle"
diff --git a/build.gradle b/build.gradle
index f598b6e83a9ca..6ca828e60ed8e 100644
--- a/build.gradle
+++ b/build.gradle
@@ -342,7 +342,8 @@ allprojects {
}
if (project.path.contains(":libs:cli")) {
// ensure we resolve p2 dependencies for the spotless eclipse formatter
- dependsOn "spotlessJavaCheck"
+ dependsOn(spotlessJavaCheck)
+ //dependsOn(spotlessJavaCheck, rewriteDryRun)
}
}
@@ -442,6 +443,7 @@ allprojects {
}
apply plugin: 'elasticsearch.formatting'
+ apply plugin: 'elasticsearch.rewrite'
}
tasks.named("updateDaemonJvm") {
diff --git a/gradle.properties b/gradle.properties
index 7c781d859cea6..2febe8537a531 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,6 +1,8 @@
org.gradle.welcome=never
org.gradle.warning.mode=none
org.gradle.parallel=true
+# https://github.com/openrewrite/rewrite-gradle-plugin/issues/212
+#org.gradle.workers.max=2
# We need to declare --add-exports to make spotless working seamlessly with jdk16
org.gradle.jvmargs=-XX:+HeapDumpOnOutOfMemoryError -Xss2m --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-opens java.base/java.time=ALL-UNNAMED
@@ -16,6 +18,8 @@ org.gradle.java.installations.auto-detect=false
# log some dependency verification info to console
org.gradle.dependency.verification.console=verbose
+# fixme undo lenient
+org.gradle.dependency.verification=lenient
# allow user to specify toolchain via the RUNTIME_JAVA_HOME environment variable
org.gradle.java.installations.fromEnv=RUNTIME_JAVA_HOME
diff --git a/gradle/build.versions.toml b/gradle/build.versions.toml
index f4f8fd1bca8ea..be7147d293a72 100644
--- a/gradle/build.versions.toml
+++ b/gradle/build.versions.toml
@@ -46,6 +46,7 @@ spock-core = { group = "org.spockframework", name="spock-core", version.ref="spo
spock-junit4 = { group = "org.spockframework", name="spock-junit4", version.ref="spock" }
spock-platform = { group = "org.spockframework", name="spock-bom", version.ref="spock" }
spotless-plugin = "com.diffplug.spotless:spotless-plugin-gradle:6.25.0"
+rewrite-plugin = "org.openrewrite.rewrite:org.openrewrite.rewrite.gradle.plugin:7.12.1"
wiremock = "com.github.tomakehurst:wiremock-jre8-standalone:2.23.2"
xmlunit-core = "org.xmlunit:xmlunit-core:2.8.2"
diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
index b3413a72885cc..840604ba30039 100644
--- a/gradle/verification-metadata.xml
+++ b/gradle/verification-metadata.xml
@@ -4221,6 +4221,11 @@
+
+
+
+
+
@@ -4231,6 +4236,11 @@
+
+
+
+
+
@@ -4446,6 +4456,16 @@
+
+
+
+
+
+
+
+
+
+