Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 23 additions & 9 deletions BUILDING.md
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,25 @@ dependencies {

To test an unreleased development version of a third party dependency you have several options.

### How do I test against java early access (ea) versions?

Currently only openjdk EA builds by oracle are supported.
To test against an early access version java version you can pass the major
java version appended with `-ea` as a system property (e.g. -Druntime.java=26-ea) to the Gradle build:

```
./gradlew clean test -Druntime.java=26-ea
```

This will run the tests using the JDK 26 EA version and pick the latest available build of the matching JDK EA version we expose
in our custom jdk catalogue at `https://storage.googleapis.com/elasticsearch-jdk-archive/jdks/openjdk/latest.json`.

To run against a specific build number of the EA build you can pass a second system property (e.g. `-Druntime.java.build=6`):

```
./gradlew clean test -Druntime.java=26-ea -Druntime.java.build=6
```

#### How to use a Maven based third party dependency via `mavenlocal`?

1. Clone the third party repository locally
Expand Down Expand Up @@ -229,11 +248,8 @@ In addition to snapshot builds JitPack supports building Pull Requests. Simply u
3. Run the Gradle build as needed. Keep in mind the initial resolution might take a bit longer as this needs to be built
by JitPack in the background before we can resolve the adhoc built dependency.

---

**NOTE**

You should only use that approach locally or on a developer branch for production dependencies as we do
> [!Note]
> You should only use that approach locally or on a developer branch for production dependencies as we do
not want to ship unreleased libraries into our releases.
---

Expand Down Expand Up @@ -265,10 +281,8 @@ allprojects {
```
4. Run the Gradle build as needed with `--write-verification-metadata` to ensure the Gradle dependency verification does not fail on your custom dependency.

---
**NOTE**

As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
> [!Note]
> As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository
that supports meta-data, it will use the second repository to provide the module.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
package org.elasticsearch.gradle.fixtures

import org.apache.commons.io.FileUtils
import org.elasticsearch.gradle.internal.test.InternalAwareGradleRunner
import org.gradle.testkit.runner.GradleRunner
import org.junit.Rule
import org.junit.rules.TemporaryFolder
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,18 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
private static final String ADOPT_JDK_VERSION_15 = "15.0.2+7"
private static final String AZUL_JDK_VERSION_8 = "8u302+b08"
private static final String AZUL_8_DISTRO_VERSION = "8.56.0.23"
private static final String CATALOG_EA_VERSION = "25-ea+30"
private static final String OPEN_JDK_VERSION = "12.0.1+99@123456789123456789123456789abcde"
private static final Pattern JDK_HOME_LOGLINE = Pattern.compile("JDK HOME: (.*)")

def setup() {
configurationCacheCompatible = false // JDK class references configurations which break configuration cache
}

@Unroll
def "jdk #jdkVendor for #platform#suffix are downloaded and extracted"() {
def "jdk #distributionVersion #jdkVendor for #platform#suffix are downloaded and extracted"() {
given:
def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform, arch);
def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform, arch, distributionVersion);
def mockedContent = filebytes(jdkVendor, platform)
buildFile.text = """
plugins {
Expand Down Expand Up @@ -77,21 +82,26 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {

where:
platform | arch | jdkVendor | jdkVersion | distributionVersion | expectedJavaBin | suffix
"linux" | "x64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | null | "bin/java" | ""
"linux" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | null | "bin/java" | ""
"linux" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | null | "bin/java" | "(old version)"
"windows" | "x64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | null | "bin/java" | ""
"windows" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | null | "bin/java" | ""
"windows" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | null | "bin/java" | "(old version)"
"darwin" | "x64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | null | "Contents/Home/bin/java" | ""
"darwin" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | null | "Contents/Home/bin/java" | ""
"darwin" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | null | "Contents/Home/bin/java" | "(old version)"
"mac" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | null | "Contents/Home/bin/java" | ""
"mac" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | null | "Contents/Home/bin/java" | "(old version)"
"darwin" | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | null | "Contents/Home/bin/java" | ""
"linux" | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | null | "bin/java" | ""
"linux" | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_11 | null | "bin/java" | "(jdk 11)"
"linux" | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_15 | null | "bin/java" | "(jdk 15)"
"linux" | "aarch64" | VENDOR_OPENJDK | CATALOG_EA_VERSION | "ea" | "bin/java" | ""
"linux" | "x64" | VENDOR_OPENJDK | CATALOG_EA_VERSION | "ea" | "bin/java" | ""
"darwin" | "aarch64" | VENDOR_OPENJDK | CATALOG_EA_VERSION | "ea" | "Contents/Home/bin/java" | ""
"darwin" | "x64" | VENDOR_OPENJDK | CATALOG_EA_VERSION | "ea" | "Contents/Home/bin/java" | ""
"windows" | "x64" | VENDOR_OPENJDK | CATALOG_EA_VERSION | "ea" | "bin/java" | ""
"linux" | "x64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | "" | "bin/java" | ""
"linux" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "" | "bin/java" | ""
"linux" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "" | "bin/java" | "(old version)"
"windows" | "x64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | "" | "bin/java" | ""
"windows" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "" | "bin/java" | ""
"windows" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "" | "bin/java" | "(old version)"
"darwin" | "x64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | "" | "Contents/Home/bin/java" | ""
"darwin" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "" | "Contents/Home/bin/java" | ""
"darwin" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "" | "Contents/Home/bin/java" | "(old version)"
"mac" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "" | "Contents/Home/bin/java" | ""
"mac" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "" | "Contents/Home/bin/java" | "(old version)"
"darwin" | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | "" | "Contents/Home/bin/java" | ""
"linux" | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | "" | "bin/java" | ""
"linux" | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_11 | "" | "bin/java" | "(jdk 11)"
"linux" | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_15 | "" | "bin/java" | "(jdk 15)"
"darwin" | "aarch64" | VENDOR_ZULU | AZUL_JDK_VERSION_8 | AZUL_8_DISTRO_VERSION | "Contents/Home/bin/java" | "(jdk 8)"
}

Expand Down Expand Up @@ -214,13 +224,19 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
private static String urlPath(final String vendor,
final String version,
final String platform,
final String arch = 'x64') {
if (vendor.equals(VENDOR_ADOPTIUM)) {
final String arch = 'x64',
final String distributedVersion = '') {
final boolean isOld = version.equals(OPENJDK_VERSION_OLD);

if (distributedVersion.equals("ea")) {
def effectivePlatform = isMac(platform) ? "macos" : platform;
def fileExtension = platform.toLowerCase().equals("windows") ? "zip" : "tar.gz";
return "/jdks/openjdk/25/openjdk-${version}/openjdk-${version}_$effectivePlatform-${arch}_bin.$fileExtension";
} else if (vendor.equals(VENDOR_ADOPTIUM)) {
final String module = isMac(platform) ? "mac" : platform;
return "/jdk-" + version + "/" + module + "/${arch}/jdk/hotspot/normal/adoptium";
} else if (vendor.equals(VENDOR_OPENJDK)) {
final String effectivePlatform = isMac(platform) ? "macos" : platform;
final boolean isOld = version.equals(OPENJDK_VERSION_OLD);
final String versionPath = isOld ? "jdk1/99" : "jdk12.0.1/123456789123456789123456789abcde/99";
final String filename = "openjdk-" + (isOld ? "1" : "12.0.1") + "_" + effectivePlatform + "-x64_bin." + extension(platform);
return "/java/GA/" + versionPath + "/GPL/" + filename;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ public class Jdk implements Buildable, Iterable<File> {
"(\\d+)(\\.\\d+\\.\\d+(?:\\.\\d+)?)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"
);
private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)\\+(b\\d+?)(@([a-f0-9]{32}))?");
private static final Pattern EA_VERSION_PATTERN = Pattern.compile("(\\d+)-ea\\+(\\d+)(@([a-f0-9]{32}))?");

private final String name;
private final FileCollection configuration;
Expand Down Expand Up @@ -78,7 +79,9 @@ public String getVersion() {
}

public void setVersion(String version) {
if (VERSION_PATTERN.matcher(version).matches() == false && LEGACY_VERSION_PATTERN.matcher(version).matches() == false) {
if (VERSION_PATTERN.matcher(version).matches() == false
&& LEGACY_VERSION_PATTERN.matcher(version).matches() == false
&& EA_VERSION_PATTERN.matcher(version).matches() == false) {
throw new IllegalArgumentException("malformed version [" + version + "] for jdk [" + name + "]");
}
parseVersion(version);
Expand Down Expand Up @@ -112,7 +115,7 @@ public void setArchitecture(final String architecture) {
}

public String getDistributionVersion() {
return distributionVersion.get();
return distributionVersion.getOrNull();
}

public void setDistributionVersion(String distributionVersion) {
Expand Down Expand Up @@ -218,9 +221,17 @@ private void parseVersion(String version) {
if (jdkVersionMatcher.matches() == false) {
// Try again with the pre-Java9 version format
jdkVersionMatcher = LEGACY_VERSION_PATTERN.matcher(version);

if (jdkVersionMatcher.matches() == false) {
throw new IllegalArgumentException("Malformed jdk version [" + version + "]");
// Try again with the pre-Java9 version format
jdkVersionMatcher = EA_VERSION_PATTERN.matcher(version);
if (jdkVersionMatcher.matches() == false) {
throw new IllegalArgumentException("Malformed jdk version [" + version + "]");
}
baseVersion = version;
major = jdkVersionMatcher.group(1);
build = jdkVersionMatcher.group(2);
hash = null;
return;
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,15 @@ private void setupRepository(Project project, Jdk jdk) {
+ jdk.getBuild()
+ "/[module]/[classifier]/jdk/hotspot/normal/adoptium";
}
} else if (jdk.getVendor().equals(VENDOR_OPENJDK) && "ea".equals(jdk.getDistributionVersion())) {
repoUrl = "https://builds.es-jdk-archive.com/";
// current pattern since 12.0.1
artifactPattern = "jdks/openjdk/"
+ jdk.getMajor()
+ "/openjdk-"
+ jdk.getBaseVersion()
+ "/"
+ "openjdk-[revision]_[module]-[classifier]_bin.[ext]";
} else if (jdk.getVendor().equals(VENDOR_OPENJDK)) {
repoUrl = "https://download.oracle.com";
if (jdk.getHash() != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,12 @@
import com.fasterxml.jackson.databind.ObjectMapper;

import org.apache.commons.io.IOUtils;
import org.elasticsearch.gradle.Architecture;
import org.elasticsearch.gradle.OS;
import org.elasticsearch.gradle.VersionProperties;
import org.elasticsearch.gradle.internal.BwcVersions;
import org.elasticsearch.gradle.internal.Jdk;
import org.elasticsearch.gradle.internal.JdkDownloadPlugin;
import org.elasticsearch.gradle.internal.conventions.GitInfoPlugin;
import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
import org.elasticsearch.gradle.internal.conventions.info.ParallelDetector;
Expand All @@ -22,6 +26,7 @@
import org.gradle.api.Action;
import org.gradle.api.GradleException;
import org.gradle.api.JavaVersion;
import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.logging.Logger;
Expand Down Expand Up @@ -63,6 +68,7 @@
import javax.inject.Inject;

import static org.elasticsearch.gradle.internal.conventions.GUtils.elvis;
import static org.elasticsearch.gradle.internal.toolchain.EarlyAccessCatalogJdkToolchainResolver.findLatestEABuildNumber;

public class GlobalBuildInfoPlugin implements Plugin<Project> {
private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class);
Expand Down Expand Up @@ -92,11 +98,13 @@ public GlobalBuildInfoPlugin(

@Override
public void apply(Project project) {

if (project != project.getRootProject()) {
throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project.");
}
this.project = project;
project.getPlugins().apply(JvmToolchainsPlugin.class);
project.getPlugins().apply(JdkDownloadPlugin.class);
Provider<GitInfo> gitInfo = project.getPlugins().apply(GitInfoPlugin.class).getGitInfo();

toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
Expand Down Expand Up @@ -270,7 +278,9 @@ private JavaVersion determineJavaVersion(String description, File javaHome, Java
private InstallationLocation getJavaInstallation(File javaHome) {
return getAvailableJavaInstallationLocationSteam().filter(installationLocation -> isSameFile(javaHome, installationLocation))
.findFirst()
.orElseThrow(() -> new GradleException("Could not locate available Java installation in Gradle registry at: " + javaHome));
.orElse(
InstallationLocation.userDefined(javaHome, "Manually resolved JavaHome (not auto-detected by Gradle toolchain service)")
);
}

private boolean isSameFile(File javaHome, InstallationLocation installationLocation) {
Expand Down Expand Up @@ -338,7 +348,14 @@ private Provider<File> findRuntimeJavaHome() {
String runtimeJavaProperty = System.getProperty("runtime.java");

if (runtimeJavaProperty != null) {
return resolveJavaHomeFromToolChainService(runtimeJavaProperty);
if (runtimeJavaProperty.toLowerCase().endsWith("-ea")) {
// handle EA builds differently due to lack of support in Gradle toolchain service
// we resolve them using JdkDownloadPlugin for now.
Integer major = Integer.parseInt(runtimeJavaProperty.substring(0, runtimeJavaProperty.length() - 3));
return resolveEarlyAccessJavaHome(major);
} else {
return resolveJavaHomeFromToolChainService(runtimeJavaProperty);
}
}
if (System.getenv("RUNTIME_JAVA_HOME") != null) {
return providers.provider(() -> new File(System.getenv("RUNTIME_JAVA_HOME")));
Expand All @@ -353,6 +370,23 @@ private Provider<File> findRuntimeJavaHome() {
});
}

private Provider<File> resolveEarlyAccessJavaHome(Integer runtimeJavaProperty) {
NamedDomainObjectContainer<Jdk> container = (NamedDomainObjectContainer<Jdk>) project.getExtensions().getByName("jdks");
Integer buildNumber = Integer.getInteger("runtime.java.build");
if (buildNumber == null) {
buildNumber = findLatestEABuildNumber(runtimeJavaProperty);
}
String eaVersionString = String.format("%d-ea+%d", runtimeJavaProperty, buildNumber);
Jdk jdk = container.create("ea_jdk_" + runtimeJavaProperty, j -> {
j.setVersion(eaVersionString);
j.setVendor("openjdk");
j.setPlatform(OS.current().javaOsReference);
j.setArchitecture(Architecture.current().javaClassifier);
j.setDistributionVersion("ea");
});
return providers.provider(() -> new File(jdk.getJavaHomePath().toString()));
}

@NotNull
private Provider<File> resolveJavaHomeFromToolChainService(String version) {
Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
Expand Down
Loading