[8.17] Revert cloud wolfi migration (elastic#119068)

This reverts the wolfi migration for cloud ess docker images.

Author: breskeby

.buildkite/pipelines/periodic-packaging.template.yml

@@ -3,7 +3,7 @@ steps:
     steps:
       - label: "{{matrix.image}} / packaging-tests-unix"
         command: ./.ci/scripts/packaging-test.sh destructivePackagingTest
-        timeout_in_minutes: 420
+        timeout_in_minutes: 300
         matrix:
           setup:
             image:

.buildkite/pipelines/periodic-packaging.yml

@@ -4,7 +4,7 @@ steps:
     steps:
       - label: "{{matrix.image}} / packaging-tests-unix"
         command: ./.ci/scripts/packaging-test.sh destructivePackagingTest
-        timeout_in_minutes: 420
+        timeout_in_minutes: 300
         matrix:
           setup:
             image:

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -5,7 +5,7 @@ steps:
     steps:
       - label: "{{matrix.image}} / docker / packaging-tests-unix"
         key: "packaging-tests-unix-docker"
-        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker-cloud-ess
+        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker
         timeout_in_minutes: 300
         matrix:
           setup:

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -21,19 +21,18 @@ public enum DockerBase {
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
     IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"),
 
+    // Based on CLOUD above, with more extras. We don't set a base image because
+    // we programmatically extend from the Cloud image.
+    CLOUD_ESS(null, "-cloud-ess", "apt-get"),
+
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
     WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:bfdeddb33330a281950c2a54adef991dbbe6a42832bc505d13b11beaf50ae73f",
         "-wolfi",
         "apk"
-    ),
+    );
     // spotless:on
-
-    // Based on WOLFI above, with more extras. We don't set a base image because
-    // we programmatically extend from the wolfi image.
-    CLOUD_ESS(null, "-cloud-ess", "apk");
-
     private final String image;
     private final String suffix;
     private final String packageManager;

distribution/docker/README.md

@@ -7,29 +7,25 @@ the [DockerBase] enum.
    * UBI - the same as the default image, but based upon [RedHat's UBI
      images][ubi], specifically their minimal flavour.
    * Wolfi - the same as the default image, but based upon [Wolfi](https://github.com/wolfi-dev)
-   * Cloud ESS - this directly extends the Wolfi image, and adds all ES plugins
-     that the ES build generates in an archive directory. It also sets an
-     environment variable that points at this directory. This allows plugins to
-     be installed from the archive instead of the internet, speeding up
-     deployment times. Furthermore this image has
-     * `filebeat` and `metricbeat` included
-     * `wget` included
-     * The `ENTRYPOINT` is just `/sbin/tini`, and the `CMD` is
-     `/app/elasticsearch.sh`. In normal use this file would be bind-mounted
-     in, but the image ships a stub version of this file so that the image
-     can still be tested.
    * Iron Bank - this is the US Department of Defence's repository of digitally
      signed, binary container images including both Free and Open-Source
      software (FOSS) and Commercial off-the-shelf (COTS). In practice, this is
      another UBI build, this time on the regular UBI image, with extra
      hardening. See below for more details.
+
    * Cloud - this is mostly the same as the default image, with some notable differences:
       * `filebeat` and `metricbeat` are included
       * `wget` is included
       * The `ENTRYPOINT` is just `/bin/tini`, and the `CMD` is
-        `/app/elasticsearch.sh`. In normal use this file would be bind-mounted
+        `/app/elasticsearc.sh`. In normal use this file would be bind-mounted
         in, but the image ships a stub version of this file so that the image
         can still be tested.
+   * Cloud ESS - this directly extends the Cloud image, and adds all ES plugins
+     that the ES build generates in an archive directory. It also sets an
+     environment variable that points at this directory. This allows plugins to
+     be installed from the archive instead of the internet, speeding up
+     deployment times.
+
 The long-term goal is for both Cloud images to be retired in favour of the
 default image.
 

distribution/docker/build.gradle

@@ -1,3 +1,4 @@
+import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.DockerBase
@@ -9,7 +10,6 @@ import org.elasticsearch.gradle.internal.docker.ShellRetry
 import org.elasticsearch.gradle.internal.docker.TransformLog4jConfigFilter
 import org.elasticsearch.gradle.internal.docker.*
 import org.elasticsearch.gradle.util.GradleUtils
-import org.elasticsearch.gradle.Architecture
 import java.nio.file.Path
 import java.time.temporal.ChronoUnit
 
@@ -99,9 +99,9 @@ String tiniArch = Architecture.current() == Architecture.AARCH64 ? 'arm64' : 'am
 
 dependencies {
   aarch64DockerSource project(":distribution:archives:linux-aarch64-tar")
-  aarch64DockerSourceTar project(path: ":distribution:archives:linux-aarch64-tar", configuration: "default")
+  aarch64DockerSourceTar project(path: ":distribution:archives:linux-aarch64-tar", configuration:"default")
   dockerSource project(":distribution:archives:linux-tar")
-  dockerSourceTar project(path: ":distribution:archives:linux-tar", configuration: "default")
+  dockerSourceTar project(path: ":distribution:archives:linux-tar", configuration:"default")
   log4jConfig project(path: ":distribution", configuration: 'log4jConfig')
   tini "krallin:tini:0.19.0:${tiniArch}"
   allPlugins project(path: ':plugins', configuration: 'allPlugins')
@@ -112,7 +112,7 @@ dependencies {
 }
 
 ext.expansions = { Architecture architecture, DockerBase base ->
-  def (major, minor) = VersionProperties.elasticsearch.split("\\.")
+  def (major,minor) = VersionProperties.elasticsearch.split("\\.")
 
   // We tag our Docker images with various pieces of information, including a timestamp
   // for when the image was built. However, this makes it impossible completely cache
@@ -216,8 +216,7 @@ elasticsearch_distributions {
 }
 
 interface Injected {
-  @Inject
-  FileSystemOperations getFs()
+  @Inject FileSystemOperations getFs()
 }
 
 tasks.named("preProcessFixture").configure {
@@ -327,9 +326,9 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
     into "${project.buildDir}/docker-context/${archiveName}"
 
     // Since we replaced the remote URL in the Dockerfile, copy in the required file
-    if (base == DockerBase.IRON_BANK) {
+    if(base == DockerBase.IRON_BANK) {
       from(architecture == Architecture.AARCH64 ? configurations.aarch64DockerSourceTar : configurations.dockerSourceTar)
-      from(configurations.tini) {
+      from (configurations.tini) {
         rename { _ -> 'tini' }
       }
     } else {
@@ -339,10 +338,7 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
     expansions(architecture, base).findAll { it.key != 'build_date' }.each { k, v ->
       inputs.property(k, { v.toString() })
     }
-    Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(
-      project.gradle.sharedServices,
-      DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
-    )
+    Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
     onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) }
   }
 
@@ -366,7 +362,7 @@ private static List<String> generateTags(DockerBase base, Architecture architect
   String image = "elasticsearch${base.suffix}"
 
   String namespace = 'elasticsearch'
-  if (base == base == DockerBase.CLOUD_ESS) {
+  if (base == DockerBase.CLOUD_ESS) {
     namespace += '-ci'
   }
 
@@ -416,10 +412,7 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
         baseImages = [base.image]
       }
 
-      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(
-        project.gradle.sharedServices,
-        DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
-      )
+      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
       onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) }
 
     }
@@ -432,12 +425,12 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
 }
 
 void addBuildEssDockerImageTask(Architecture architecture) {
-  DockerBase dockerBase = DockerBase.CLOUD_ESS
+  DockerBase base = DockerBase.CLOUD_ESS
   String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
-  String contextDir = "${project.buildDir}/docker-context/elasticsearch${dockerBase.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}"
+  String contextDir = "${project.buildDir}/docker-context/elasticsearch${base.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}"
 
   final TaskProvider<Sync> buildContextTask =
-    tasks.register(taskName('build', architecture, dockerBase, 'DockerContext'), Sync) {
+    tasks.register(taskName('build', architecture, base, 'DockerContext'), Sync) {
       into contextDir
 
       final Path projectDir = project.projectDir.toPath()
@@ -446,52 +439,28 @@ void addBuildEssDockerImageTask(Architecture architecture) {
         from configurations.allPlugins
       }
 
-      // If we're performing a release build, but `build.id` hasn't been set, we can
-      // infer that we're not at the Docker building stage of the build, and therefore
-      // we should skip the beats part of the build.
-      String buildId = providers.systemProperty('build.id').getOrNull()
-      boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
-
-      if (includeBeats) {
-        from configurations.getByName("filebeat_${architecture.classifier}")
-        from configurations.getByName("metricbeat_${architecture.classifier}")
-      }
-      // For some reason, the artifact name can differ depending on what repository we used.
-      rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
-
-      String baseSuffix = DockerBase.WOLFI.suffix
-      from(projectDir.resolve("src/docker/Dockerfile.ess")) {
-        expand(
-          [
-            base_image: "elasticsearch${baseSuffix}:${architecture.classifier}",
-            docker_base: "${dockerBase.name().toLowerCase()}",
-            version: "${VersionProperties.elasticsearch}",
-            retry: ShellRetry
-          ]
-        )
+      from(projectDir.resolve("src/docker/Dockerfile.cloud-ess")) {
+        expand([
+          base_image: "elasticsearch${DockerBase.CLOUD.suffix}:${architecture.classifier}"
+        ])
         filter SquashNewlinesFilter
-        rename ~/Dockerfile\.ess$/, 'Dockerfile'
+        rename ~/Dockerfile\.cloud-ess$/, 'Dockerfile'
       }
     }
 
   final TaskProvider<DockerBuildTask> buildDockerImageTask =
-    tasks.register(taskName("build", architecture, dockerBase, "DockerImage"), DockerBuildTask) {
-
-      DockerBase base = DockerBase.WOLFI
+    tasks.register(taskName("build", architecture, base, "DockerImage"), DockerBuildTask) {
 
-      TaskProvider<DockerBuildTask> buildBaseTask = tasks.named(taskName("build", architecture, base, "DockerImage"))
-      inputs.files(buildBaseTask)
+      TaskProvider<DockerBuildTask> buildCloudTask = tasks.named(taskName("build", architecture, DockerBase.CLOUD, "DockerImage"))
+      inputs.files(buildCloudTask)
 
       dockerContext.fileProvider(buildContextTask.map { it.getDestinationDir() })
 
       noCache = buildParams.isCi()
       baseImages = []
-      tags = generateTags(dockerBase, architecture)
+      tags = generateTags(base, architecture)
       platforms.add(architecture.dockerPlatform)
-      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(
-        project.gradle.sharedServices,
-        DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
-      )
+      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
       onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) }
 
     }
@@ -542,7 +511,7 @@ subprojects { Project subProject ->
       (base == DockerBase.IRON_BANK ? 'ironbank.tar' :
           (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' :
             (base == DockerBase.WOLFI ? 'wolfi.tar' :
-                'docker.tar')))
+              'docker.tar')))
     final String artifactName = "elasticsearch${arch}${base.suffix}_test"
 
     final String exportTaskName = taskName("export", architecture, base, 'DockerImage')
@@ -558,10 +527,7 @@ subprojects { Project subProject ->
         tarFile,
         "elasticsearch${base.suffix}:${architecture.classifier}"
       dependsOn(parent.path + ":" + buildTaskName)
-      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(
-        project.gradle.sharedServices,
-        DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
-      )
+      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
       onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) }
     }
 

distribution/docker/src/docker/Dockerfile.cloud-ess

@@ -0,0 +1,13 @@
+FROM ${base_image} AS builder
+
+USER root
+
+COPY plugins/*.zip /opt/plugins/archive/
+
+RUN chown root.root /opt/plugins/archive/*
+RUN chmod 0444 /opt/plugins/archive/*
+
+FROM ${base_image}
+
+COPY --from=builder /opt/plugins /opt/plugins
+ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive

distribution/docker/src/docker/Dockerfile.ess

@@ -99,7 +99,6 @@
  *     <li>The default image with a custom, small base image</li>
  *     <li>A UBI-based image</li>
  *     <li>Another UBI image for Iron Bank</li>
- *     <li>A WOLFI-based image</li>
  *     <li>Images for Cloud</li>
  * </ul>
  */
@@ -204,9 +203,7 @@ public void test022InstallPluginsFromLocalArchive() {
         final String plugin = "analysis-icu";
         final Installation.Executables bin = installation.executables();
 
-        listPluginArchive().forEach(System.out::println);
         assertThat("Expected " + plugin + " to not be installed", listPlugins(), not(hasItems(plugin)));
-        assertThat("Expected " + plugin + " available in archive", listPluginArchive(), hasSize(16));
 
         // Stuff the proxy settings with garbage, so any attempt to go out to the internet would fail
         sh.getEnv()
@@ -386,7 +383,7 @@ public void test040JavaUsesTheOsProvidedKeystore() {
         if (distribution.packaging == Packaging.DOCKER_UBI || distribution.packaging == Packaging.DOCKER_IRON_BANK) {
             // In these images, the `cacerts` file ought to be a symlink here
             assertThat(path, equalTo("/etc/pki/ca-trust/extracted/java/cacerts"));
-        } else if (distribution.packaging == Packaging.DOCKER_WOLFI || distribution.packaging == Packaging.DOCKER_CLOUD_ESS) {
+        } else if (distribution.packaging == Packaging.DOCKER_WOLFI) {
             // In these images, the `cacerts` file ought to be a symlink here
             assertThat(path, equalTo("/etc/ssl/certs/java/cacerts"));
         } else {
@@ -1113,7 +1110,7 @@ public void test170DefaultShellIsBash() {
      */
     public void test171AdditionalCliOptionsAreForwarded() throws Exception {
         assumeTrue(
-            "Does not apply to Cloud ESS images, because they don't use the default entrypoint",
+            "Does not apply to Cloud ess images, because they don't use the default entrypoint",
             distribution().packaging != Packaging.DOCKER_CLOUD_ESS
         );
 
@@ -1219,10 +1216,6 @@ private List<String> listPlugins() {
         return sh.run(bin.pluginTool + " list").stdout().lines().collect(Collectors.toList());
     }
 
-    private List<String> listPluginArchive() {
-        return sh.run("ls -lh /opt/plugins/archive").stdout().lines().collect(Collectors.toList());
-    }
-
     /**
      * Check that readiness listener works
      */