diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e5ddd80ecf..f9cf1253a7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -287,6 +287,24 @@ stages: fetchDepth: 1 clean: false persistCredentials: true + - bash: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/.ghcup + sudo rm -rf /usr/share/swift + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /opt/hostedtoolcache + sudo rm -rf /usr/share/rust + sudo rm -rf /usr/share/miniconda + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /usr/local/share/chrome_driver + sudo rm -rf /opt/pipx + docker system prune -af + docker builder prune -af + df -h / + displayName: 'Free up disk space and clean Docker' + target: host - task: Cache@2 displayName: 'Restore Gradle cache' inputs: @@ -338,6 +356,24 @@ stages: fetchDepth: 1 clean: false persistCredentials: true + - bash: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/.ghcup + sudo rm -rf /usr/share/swift + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /opt/hostedtoolcache + sudo rm -rf /usr/share/rust + sudo rm -rf /usr/share/miniconda + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /usr/local/share/chrome_driver + sudo rm -rf /opt/pipx + docker system prune -af + docker builder prune -af + df -h / + displayName: 'Free up disk space and clean Docker' + target: host - task: Cache@2 displayName: 'Restore Gradle cache' inputs: @@ -417,6 +453,24 @@ stages: fetchDepth: 1 clean: false persistCredentials: true + - bash: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/.ghcup + sudo rm -rf /usr/share/swift + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /opt/hostedtoolcache + sudo rm -rf /usr/share/rust + sudo rm -rf /usr/share/miniconda + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /usr/local/share/chrome_driver + sudo rm -rf /opt/pipx + docker system prune -af + docker builder prune -af + df -h / + displayName: 'Free up disk space and clean Docker' + target: host - task: Cache@2 displayName: 'Restore Gradle cache' inputs: @@ -486,6 +540,24 @@ stages: fetchDepth: 1 clean: false persistCredentials: true + - bash: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/.ghcup + sudo rm -rf /usr/share/swift + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /opt/hostedtoolcache + sudo rm -rf /usr/share/rust + sudo rm -rf /usr/share/miniconda + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /usr/local/share/chrome_driver + sudo rm -rf /opt/pipx + docker system prune -af + docker builder prune -af + df -h / + displayName: 'Free up disk space and clean Docker' + target: host - task: Cache@2 displayName: 'Restore Gradle cache' inputs: @@ -533,6 +605,24 @@ stages: fetchDepth: 1 clean: false persistCredentials: true + - bash: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/.ghcup + sudo rm -rf /usr/share/swift + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /opt/hostedtoolcache + sudo rm -rf /usr/share/rust + sudo rm -rf /usr/share/miniconda + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /usr/local/share/chrome_driver + sudo rm -rf /opt/pipx + docker system prune -af + docker builder prune -af + df -h / + displayName: 'Free up disk space and clean Docker' + target: host - task: Cache@2 displayName: 'Restore Gradle cache' inputs: @@ -584,6 +674,24 @@ stages: fetchDepth: 1 clean: false persistCredentials: true + - bash: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/.ghcup + sudo rm -rf /usr/share/swift + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /opt/hostedtoolcache + sudo rm -rf /usr/share/rust + sudo rm -rf /usr/share/miniconda + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /usr/local/share/chrome_driver + sudo rm -rf /opt/pipx + docker system prune -af + docker builder prune -af + df -h / + displayName: 'Free up disk space and clean Docker' + target: host - task: Cache@2 displayName: 'Restore Gradle cache' inputs: diff --git a/es716x/Dockerfile b/es716x/Dockerfile index 9b5323857e..0dc2a61ee0 100644 --- a/es716x/Dockerfile +++ b/es716x/Dockerfile @@ -32,7 +32,15 @@ USER root RUN chmod +x /usr/local/bin/gosu \ && gosu --version \ - && gosu nobody true + && gosu nobody true \ + # ES 7.16.x bundles JDK 17.0.1 which crashes on cgroup v2 hosts (JDK-8287073). \ + # Replace with Corretto 17.0.5 which contains the fix (backport JDK-8288308). \ + && ARCH=$(uname -m | sed 's/x86_64/x64/' | sed 's/arm64/aarch64/') \ + && curl -fsSLk "https://corretto.aws/downloads/resources/17.0.5.8.1/amazon-corretto-17.0.5.8.1-linux-${ARCH}.tar.gz" -o /tmp/jdk.tar.gz \ + && rm -rf /usr/share/elasticsearch/jdk \ + && mkdir -p /usr/share/elasticsearch/jdk \ + && tar xzf /tmp/jdk.tar.gz -C /usr/share/elasticsearch/jdk --strip-components=1 \ + && rm /tmp/jdk.tar.gz USER elasticsearch diff --git a/es717x/Dockerfile b/es717x/Dockerfile index 9b5323857e..4b8e317d39 100644 --- a/es717x/Dockerfile +++ b/es717x/Dockerfile @@ -32,7 +32,27 @@ USER root RUN chmod +x /usr/local/bin/gosu \ && gosu --version \ - && gosu nobody true + && gosu nobody true \ + # ES 7.17.0–7.17.2 bundle JDK 17.0.x and 7.17.3–7.17.6 bundle JDK 18, both of which \ + # crash on cgroup v2 hosts (JDK-8287073). Replace with Corretto 17.0.5 (fix backport \ + # JDK-8288308) for the JDK-17 builds, or Corretto 19.0.0 (fixed) for the JDK-18 builds. \ + # 7.17.7+ bundle JDK 19+ which already contains the fix, so no replacement is needed. \ + && PATCH=$(echo "$ES_VERSION" | cut -d. -f3) \ + && if [ "$PATCH" -le 2 ]; then \ + ARCH=$(uname -m | sed 's/x86_64/x64/' | sed 's/arm64/aarch64/') \ + && curl -fsSLk "https://corretto.aws/downloads/resources/17.0.5.8.1/amazon-corretto-17.0.5.8.1-linux-${ARCH}.tar.gz" -o /tmp/jdk.tar.gz \ + && rm -rf /usr/share/elasticsearch/jdk \ + && mkdir -p /usr/share/elasticsearch/jdk \ + && tar xzf /tmp/jdk.tar.gz -C /usr/share/elasticsearch/jdk --strip-components=1 \ + && rm /tmp/jdk.tar.gz; \ + elif [ "$PATCH" -le 6 ]; then \ + ARCH=$(uname -m | sed 's/x86_64/x64/' | sed 's/arm64/aarch64/') \ + && curl -fsSLk "https://corretto.aws/downloads/resources/19.0.0.36.1/amazon-corretto-19.0.0.36.1-linux-${ARCH}.tar.gz" -o /tmp/jdk.tar.gz \ + && rm -rf /usr/share/elasticsearch/jdk \ + && mkdir -p /usr/share/elasticsearch/jdk \ + && tar xzf /tmp/jdk.tar.gz -C /usr/share/elasticsearch/jdk --strip-components=1 \ + && rm /tmp/jdk.tar.gz; \ + fi USER elasticsearch diff --git a/es80x/Dockerfile b/es80x/Dockerfile index 1684650a66..be763e9995 100644 --- a/es80x/Dockerfile +++ b/es80x/Dockerfile @@ -32,7 +32,15 @@ USER root RUN chmod +x /usr/local/bin/gosu \ && gosu --version \ - && gosu nobody true + && gosu nobody true \ + # ES 8.0.x bundles JDK 17.0.2 which crashes on cgroup v2 hosts (JDK-8287073). \ + # Replace with Corretto 17.0.5 which contains the fix (backport JDK-8288308). \ + && ARCH=$(uname -m | sed 's/x86_64/x64/' | sed 's/arm64/aarch64/') \ + && curl -fsSLk "https://corretto.aws/downloads/resources/17.0.5.8.1/amazon-corretto-17.0.5.8.1-linux-${ARCH}.tar.gz" -o /tmp/jdk.tar.gz \ + && rm -rf /usr/share/elasticsearch/jdk \ + && mkdir -p /usr/share/elasticsearch/jdk \ + && tar xzf /tmp/jdk.tar.gz -C /usr/share/elasticsearch/jdk --strip-components=1 \ + && rm /tmp/jdk.tar.gz USER elasticsearch diff --git a/es81x/Dockerfile b/es81x/Dockerfile index 1684650a66..4f4ce06951 100644 --- a/es81x/Dockerfile +++ b/es81x/Dockerfile @@ -32,7 +32,15 @@ USER root RUN chmod +x /usr/local/bin/gosu \ && gosu --version \ - && gosu nobody true + && gosu nobody true \ + # ES 8.1.x bundles JDK 17.0.2 which crashes on cgroup v2 hosts (JDK-8287073). \ + # Replace with Corretto 17.0.5 which contains the fix (backport JDK-8288308). \ + && ARCH=$(uname -m | sed 's/x86_64/x64/' | sed 's/arm64/aarch64/') \ + && curl -fsSLk "https://corretto.aws/downloads/resources/17.0.5.8.1/amazon-corretto-17.0.5.8.1-linux-${ARCH}.tar.gz" -o /tmp/jdk.tar.gz \ + && rm -rf /usr/share/elasticsearch/jdk \ + && mkdir -p /usr/share/elasticsearch/jdk \ + && tar xzf /tmp/jdk.tar.gz -C /usr/share/elasticsearch/jdk --strip-components=1 \ + && rm /tmp/jdk.tar.gz USER elasticsearch diff --git a/es82x/Dockerfile b/es82x/Dockerfile index 1684650a66..93a920bc6f 100644 --- a/es82x/Dockerfile +++ b/es82x/Dockerfile @@ -32,7 +32,15 @@ USER root RUN chmod +x /usr/local/bin/gosu \ && gosu --version \ - && gosu nobody true + && gosu nobody true \ + # ES 8.2.x bundles JDK 18.0.x which crashes on cgroup v2 hosts (JDK-8287073). \ + # Replace with Corretto 19.0.0 which contains the fix. \ + && ARCH=$(uname -m | sed 's/x86_64/x64/' | sed 's/arm64/aarch64/') \ + && curl -fsSLk "https://corretto.aws/downloads/resources/19.0.0.36.1/amazon-corretto-19.0.0.36.1-linux-${ARCH}.tar.gz" -o /tmp/jdk.tar.gz \ + && rm -rf /usr/share/elasticsearch/jdk \ + && mkdir -p /usr/share/elasticsearch/jdk \ + && tar xzf /tmp/jdk.tar.gz -C /usr/share/elasticsearch/jdk --strip-components=1 \ + && rm /tmp/jdk.tar.gz USER elasticsearch diff --git a/es83x/Dockerfile b/es83x/Dockerfile index 1684650a66..da1a0ad117 100644 --- a/es83x/Dockerfile +++ b/es83x/Dockerfile @@ -32,7 +32,15 @@ USER root RUN chmod +x /usr/local/bin/gosu \ && gosu --version \ - && gosu nobody true + && gosu nobody true \ + # ES 8.3.x bundles JDK 18.0.x which crashes on cgroup v2 hosts (JDK-8287073). \ + # Replace with Corretto 19.0.0 which contains the fix. \ + && ARCH=$(uname -m | sed 's/x86_64/x64/' | sed 's/arm64/aarch64/') \ + && curl -fsSLk "https://corretto.aws/downloads/resources/19.0.0.36.1/amazon-corretto-19.0.0.36.1-linux-${ARCH}.tar.gz" -o /tmp/jdk.tar.gz \ + && rm -rf /usr/share/elasticsearch/jdk \ + && mkdir -p /usr/share/elasticsearch/jdk \ + && tar xzf /tmp/jdk.tar.gz -C /usr/share/elasticsearch/jdk --strip-components=1 \ + && rm /tmp/jdk.tar.gz USER elasticsearch diff --git a/es84x/Dockerfile b/es84x/Dockerfile index 1684650a66..0b694324e9 100644 --- a/es84x/Dockerfile +++ b/es84x/Dockerfile @@ -32,7 +32,15 @@ USER root RUN chmod +x /usr/local/bin/gosu \ && gosu --version \ - && gosu nobody true + && gosu nobody true \ + # ES 8.4.x bundles JDK 18.0.x which crashes on cgroup v2 hosts (JDK-8287073). \ + # Replace with Corretto 19.0.0 which contains the fix. \ + && ARCH=$(uname -m | sed 's/x86_64/x64/' | sed 's/arm64/aarch64/') \ + && curl -fsSLk "https://corretto.aws/downloads/resources/19.0.0.36.1/amazon-corretto-19.0.0.36.1-linux-${ARCH}.tar.gz" -o /tmp/jdk.tar.gz \ + && rm -rf /usr/share/elasticsearch/jdk \ + && mkdir -p /usr/share/elasticsearch/jdk \ + && tar xzf /tmp/jdk.tar.gz -C /usr/share/elasticsearch/jdk --strip-components=1 \ + && rm /tmp/jdk.tar.gz USER elasticsearch diff --git a/tests-utils/src/main/scala/tech/beshu/ror/utils/containers/images/DockerImageDescription.scala b/tests-utils/src/main/scala/tech/beshu/ror/utils/containers/images/DockerImageDescription.scala index dc4708cf69..0ce929f5a4 100644 --- a/tests-utils/src/main/scala/tech/beshu/ror/utils/containers/images/DockerImageDescription.scala +++ b/tests-utils/src/main/scala/tech/beshu/ror/utils/containers/images/DockerImageDescription.scala @@ -31,6 +31,10 @@ final case class DockerImageDescription(baseImage: String, this.copy(runCommands = this.runCommands :+ Command.Run(command)) } + def run(command: String, otherCommands: String*): DockerImageDescription = { + this.copy(runCommands = (this.runCommands :+ Command.Run(command)) ++ otherCommands.map(Command.Run.apply)) + } + def runWhen(condition: Boolean, command: => String): DockerImageDescription = { if (condition) run(command) else this diff --git a/tests-utils/src/main/scala/tech/beshu/ror/utils/containers/images/Elasticsearch.scala b/tests-utils/src/main/scala/tech/beshu/ror/utils/containers/images/Elasticsearch.scala index 12e138141b..7f2d0d9f8b 100644 --- a/tests-utils/src/main/scala/tech/beshu/ror/utils/containers/images/Elasticsearch.scala +++ b/tests-utils/src/main/scala/tech/beshu/ror/utils/containers/images/Elasticsearch.scala @@ -24,7 +24,8 @@ import tech.beshu.ror.utils.containers.ContainerUtils import tech.beshu.ror.utils.containers.images.Elasticsearch.* import tech.beshu.ror.utils.containers.images.Elasticsearch.Plugin.{PluginInstallationStep, PluginInstallationSteps} import tech.beshu.ror.utils.containers.windows.WindowsEsDirectoryManager -import tech.beshu.ror.utils.misc.Version +import tech.beshu.ror.utils.misc.{JDK, Version} + object Elasticsearch { @@ -183,6 +184,7 @@ class Elasticsearch(val esVersion: String, // Package tar is required by the RorToolsAppSuite, and the ES >= 9.x is based on // Red Hat Universal Base Image 9 Minimal, which does not contain it. .runWhen(Version.greaterOrEqualThan(esVersion, 9, 0, 0), "microdnf install -y tar") + .when(hasBuggyBundledJdk, replaceBundledJdk) .run(s"chown -R elasticsearch:elasticsearch ${config.esConfigDir.toString()}") .addEnvs(config.envs + ("ES_JAVA_OPTS" -> javaOptsBasedOn(withEsJavaOptsBuilderFromPlugins))) .installPlugins() @@ -200,6 +202,7 @@ class Elasticsearch(val esVersion: String, .run(s"""echo "deb https://artifacts.elastic.co/packages/$esMajorVersion/apt stable main" > /etc/apt/sources.list.d/elastic-$esMajorVersion.list""") .run(s"""apt update && apt install -y --no-install-recommends -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" elasticsearch=$esVersion""") .run("apt clean && rm -rf /var/lib/apt/lists/*") + .when(hasBuggyBundledJdk, replaceBundledJdk) .user("elasticsearch") .setCommand("/usr/share/elasticsearch/bin/elasticsearch") .copyFile( @@ -303,6 +306,37 @@ class Elasticsearch(val esVersion: String, ) } + // ES 7.16.x–7.17.6 and 8.0.x–8.4.x bundle JDK 17.0.1/17.0.2 or JDK 18, which have cgroup v2 + // bug JDK-8287073: CgroupV2Subsystem.getInstance() NPEs before UseContainerSupport is checked. + // Fixed in JDK 17.0.5+ (backport JDK-8288308) and JDK 19+. + // NOTE: the same version-range logic is maintained in the e2e-tests repo at + // environments/common/images/es-jdk-patch/patch-es-jdk.sh + private def hasBuggyBundledJdk: Boolean = + (Version.greaterOrEqualThan(esVersion, 7, 16, 0) && Version.lowerThan(esVersion, 7, 17, 7)) || + (Version.greaterOrEqualThan(esVersion, 8, 0, 0) && Version.lowerThan(esVersion, 8, 5, 0)) + + // Replace the bundled JDK in-place with the cached custom JDK tarball. + private def replaceBundledJdk(image: DockerImageDescription): DockerImageDescription = { + // JDK 17.0.0–17.0.4 and JDK 18 have cgroup v2 bug JDK-8287073 (NPE in CgroupV2Subsystem). + // JDK-17 builds (ES 7.16.x, 7.17.0–7.17.2, 8.0.x–8.1.x) → Corretto 17.0.5 (backport JDK-8288308). + // JDK-18 builds (ES 7.17.3–7.17.6, 8.2.x–8.4.x) → Corretto 19.0.0 (first JDK 19 with the fix). + // Downloaded once per JVM process and reused across all container builds. + val needsCorretto19 = + (Version.greaterOrEqualThan(esVersion, 7, 17, 3) && Version.lowerThan(esVersion, 7, 17, 7)) || + (Version.greaterOrEqualThan(esVersion, 8, 2, 0) && Version.lowerThan(esVersion, 8, 5, 0)) + val tarball = + if (needsCorretto19) JDK.AmazonCorretto1900jdk.tarball + else JDK.AmazonCorretto1705jdk.tarball + image + .copyFile(destination = os.root / "tmp" / "custom-jdk.tar.gz", file = tarball) + .run( + "rm -rf /usr/share/elasticsearch/jdk", + "mkdir -p /usr/share/elasticsearch/jdk", + "tar xzf /tmp/custom-jdk.tar.gz -C /usr/share/elasticsearch/jdk --strip-components=1", + "rm /tmp/custom-jdk.tar.gz" + ) + } + private def javaOptsBasedOn(withEsJavaOptsBuilder: EsJavaOptsBuilder => EsJavaOptsBuilder) = { withEsJavaOptsBuilder(baseJavaOptsBuilder) .options diff --git a/tests-utils/src/main/scala/tech/beshu/ror/utils/misc/JDK.scala b/tests-utils/src/main/scala/tech/beshu/ror/utils/misc/JDK.scala new file mode 100644 index 0000000000..cd83c91915 --- /dev/null +++ b/tests-utils/src/main/scala/tech/beshu/ror/utils/misc/JDK.scala @@ -0,0 +1,51 @@ +/* + * This file is part of ReadonlyREST. + * + * ReadonlyREST is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * ReadonlyREST is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with ReadonlyREST. If not, see http://www.gnu.org/licenses/ + */ +package tech.beshu.ror.utils.misc + +import better.files.* +import com.typesafe.scalalogging.LazyLogging + +object JDK extends LazyLogging { + + object AmazonCorretto1705jdk { + lazy val tarball: File = downloadCorretto("17.0.5.8.1") + } + + object AmazonCorretto1900jdk { + lazy val tarball: File = downloadCorretto("19.0.0.36.1") + } + + private def downloadCorretto(version: String): File = { + val majorVersion = version.takeWhile(_ != '.') + val arch = System.getProperty("os.arch") match { + case a if a == "aarch64" || a == "arm64" => "aarch64" + case _ => "x64" + } + val targetFile = File.newTemporaryFile(s"amazon-corretto-$majorVersion-jdk", ".tar.gz") + logger.info(s"Downloading Amazon Corretto $majorVersion JDK (one-time, for replacing buggy bundled JDK)...") + val url = new java.net.URL(s"https://corretto.aws/downloads/resources/$version/amazon-corretto-$version-linux-$arch.tar.gz") + val connection = url.openConnection() + connection.setConnectTimeout(30_000) + connection.setReadTimeout(120_000) + for { + in <- connection.getInputStream.autoClosed + out <- targetFile.newOutputStream.autoClosed + } in.pipeTo(out) + logger.info(s"Downloaded Amazon Corretto $majorVersion JDK to ${targetFile.pathAsString}") + targetFile + } +}