diff --git a/.ci/scripts/packaging-test.sh b/.ci/scripts/packaging-test.sh index 10e20d9138bb0..b7f506022f48d 100755 --- a/.ci/scripts/packaging-test.sh +++ b/.ci/scripts/packaging-test.sh @@ -3,7 +3,7 @@ # opensuse 15 has a missing dep for systemd if which zypper > /dev/null ; then - sudo zypper install -y insserv-compat + sudo zypper install -y insserv-compat docker-buildx fi if [ -e /etc/sysctl.d/99-gce.conf ]; then diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index 7badb42f89bbe..63552a2ed5e56 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -13,13 +13,13 @@ * This class models the different Docker base images that are used to build Docker distributions of Elasticsearch. */ public enum DockerBase { - DEFAULT("ubuntu:20.04", "", "apt-get"), + DEFAULT("ubuntu:20.04", "", "apt-get", "Dockerfile"), // "latest" here is intentional, since the image name specifies "8" - UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi8", "microdnf"), + UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi8", "microdnf", "Dockerfile"), // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build - IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"), + IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum", "Dockerfile"), // Chainguard based wolfi image with latest jdk // This is usually updated via renovatebot @@ -27,31 +27,36 @@ public enum DockerBase { WOLFI( "docker.elastic.co/wolfi/chainguard-base:latest@sha256:1c7f5aa0e7997455b8500d095c7a90e617102d3941eb0757ac62cfea509e09b9", "-wolfi", - "apk" + "apk", + "Dockerfile" ), // spotless:on // Based on WOLFI above, with more extras. We don't set a base image because // we programmatically extend from the wolfi image. - CLOUD_ESS(null, "-cloud-ess", "apk"), + CLOUD_ESS(null, "-cloud-ess", "apk", "Dockerfile.ess"), + CLOUD_ESS_FIPS( "docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7", "-cloud-ess-fips", - "apk" + "apk", + "Dockerfile.ess-fips" ); private final String image; private final String suffix; private final String packageManager; + private final String dockerfile; DockerBase(String image, String suffix) { - this(image, suffix, "apt-get"); + this(image, suffix, "apt-get", "dockerfile"); } - DockerBase(String image, String suffix, String packageManager) { + DockerBase(String image, String suffix, String packageManager, String dockerfile) { this.image = image; this.suffix = suffix; this.packageManager = packageManager; + this.dockerfile = dockerfile; } public String getImage() { @@ -65,4 +70,8 @@ public String getSuffix() { public String getPackageManager() { return packageManager; } + + public String getDockerfile() { + return dockerfile; + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java index 9b28401994ee2..728d6140ee019 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java @@ -170,6 +170,7 @@ private void pullBaseImage(String baseImage) { maybeConfigureDockerConfig(spec); spec.executable("docker"); spec.args("pull"); + spec.environment("DOCKER_BUILDKIT", "1"); spec.args(baseImage); }); @@ -205,7 +206,7 @@ public void execute() { maybeConfigureDockerConfig(spec); spec.executable("docker"); - + spec.environment("DOCKER_BUILDKIT", "1"); if (isCrossPlatform) { spec.args("buildx"); } diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 70b580950eacd..b59644411bd98 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -201,9 +201,10 @@ ext.dockerBuildContext = { Architecture architecture, DockerBase base -> from projectDir.resolve("src/docker/config") } } - from(projectDir.resolve("src/docker/Dockerfile")) { + from(projectDir.resolve("src/docker/${base.dockerfile}")) { expand(varExpansions) filter SquashNewlinesFilter + rename base.dockerfile, "Dockerfile" } } } diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 8f5685a41612c..9470f5a336795 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -43,13 +43,13 @@ RUN chmod 0555 /bin/tini # Install required packages to extract the Elasticsearch distribution <% if (docker_base == 'default') { %> RUN <%= retry.loop(package_manager, "${package_manager} update && DEBIAN_FRONTEND=noninteractive ${package_manager} install -y curl ") %> -<% } else if (docker_base == "wolfi" || docker_base == 'cloud_ess_fips') { %> +<% } else if (docker_base == "wolfi") { %> RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %> <% } else { %> RUN <%= retry.loop(package_manager, "${package_manager} install -y findutils tar gzip") %> <% } %> -<% if (docker_base != 'wolfi' && docker_base != 'cloud_ess_fips' ) { %> +<% if (docker_base != 'wolfi') { %> # `tini` is a tiny but valid init for containers. This is used to cleanly # control how ES and any child processes are shut down. # For wolfi we pick it from the blessed wolfi package registry. @@ -117,47 +117,6 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas chmod 0775 bin config config/jvm.options.d data logs plugins && \\ find config -type f -exec chmod 0664 {} + -<% if (docker_base == 'cloud_ess_fips') { %> -# Add plugins infrastructure -RUN mkdir -p /opt/plugins/archive -RUN chmod -R 0555 /opt/plugins - -RUN mkdir -p /fips/libs -COPY fips/libs/*.jar /fips/libs/ - -COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/ -RUN set -eux ; \\ - for beat in filebeat metricbeat ; do \\ - if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\ - echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\ - exit 1 ; \\ - fi ; \\ - if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\ - echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\ - exit 1 ; \\ - fi ; \\ - mkdir -p /opt/\$beat ; \\ - tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\ - done - -COPY plugins/*.zip /opt/plugins/archive/ - -RUN chown 1000:1000 /opt/plugins/archive/* -RUN chmod 0444 /opt/plugins/archive/* - -COPY fips/resources/fips_java.security /usr/share/elasticsearch/config/fips_java.security -COPY fips/resources/fips_java.policy /usr/share/elasticsearch/config/fips_java.policy - -WORKDIR /usr/share/elasticsearch/config - -## Add fips specific JVM options -RUN cat < /usr/share/elasticsearch/config/jvm.options.d/fips.options --Djavax.net.ssl.keyStoreType=BCFKS --Dorg.bouncycastle.fips.approved_only=true --Djava.security.properties=config/fips_java.security --Djava.security.policy=config/fips_java.policy -EOF -<% } %> ################################################################################ # Build stage 2 (the actual Elasticsearch image): @@ -178,7 +137,7 @@ RUN ${package_manager} update --setopt=tsflags=nodocs -y && \\ nc shadow-utils zip findutils unzip procps-ng && \\ ${package_manager} clean all -<% } else if (docker_base == "wolfi" || docker_base == 'cloud_ess_fips') { %> +<% } else if (docker_base == "wolfi") { %> RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && \n" + " ${package_manager} update && \n" + @@ -227,7 +186,7 @@ RUN groupadd -g 1000 elasticsearch && \\ adduser --uid 1000 --gid 1000 --home /usr/share/elasticsearch elasticsearch && \\ adduser elasticsearch root && \\ chown -R 0:0 /usr/share/elasticsearch -<% } else if (docker_base == "wolfi" || docker_base == 'cloud_ess_fips') { %> +<% } else if (docker_base == "wolfi") { %> RUN groupadd -g 1000 elasticsearch && \ adduser -G elasticsearch -u 1000 elasticsearch -D --home /usr/share/elasticsearch elasticsearch && \ adduser elasticsearch root && \ @@ -243,7 +202,7 @@ ENV ELASTIC_CONTAINER true WORKDIR /usr/share/elasticsearch COPY --from=builder --chown=0:0 /usr/share/elasticsearch /usr/share/elasticsearch -<% if (docker_base != "wolfi" && docker_base != 'cloud_ess_fips') { %> +<% if (docker_base != "wolfi") { %> COPY --from=builder --chown=0:0 /bin/tini /bin/tini <% } %> @@ -277,7 +236,7 @@ RUN chmod g=u /etc/passwd && \\ # stays up-to-date with changes to Ubuntu's store) COPY bin/docker-openjdk /etc/ca-certificates/update.d/docker-openjdk RUN /etc/ca-certificates/update.d/docker-openjdk -<% } else if (docker_base == 'wolfi' || docker_base == 'cloud_ess_fips') { %> +<% } else if (docker_base == 'wolfi') { %> RUN ln -sf /etc/ssl/certs/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts <% } else { %> RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts @@ -324,7 +283,7 @@ RUN mkdir /licenses && cp LICENSE.txt /licenses/LICENSE COPY LICENSE /licenses/LICENSE.addendum <% } %> -<% if (docker_base == "wolfi" || docker_base == 'cloud_ess_fips') { %> +<% if (docker_base == "wolfi") { %> # Our actual entrypoint is `tini`, a minimal but functional init program. It # calls the entrypoint we provide, while correctly forwarding signals. ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"] @@ -343,13 +302,6 @@ USER 1000:0 <% if (docker_base == 'iron_bank') { %> HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 http://localhost:9200 || exit 1 <% } %> - -<% if (docker_base == 'cloud_ess_fips') { %> -COPY --from=builder --chown=0:0 /opt /opt -ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive -WORKDIR /usr/share/elasticsearch -COPY --from=builder --chown=0:0 /fips/libs/*.jar /usr/share/elasticsearch/lib/ -<% } %> ################################################################################ # End of multi-stage Dockerfile -################################################################################ \ No newline at end of file +################################################################################ diff --git a/distribution/docker/src/docker/Dockerfile.ess-fips b/distribution/docker/src/docker/Dockerfile.ess-fips new file mode 100644 index 0000000000000..58ecf45820d63 --- /dev/null +++ b/distribution/docker/src/docker/Dockerfile.ess-fips @@ -0,0 +1,206 @@ +################################################################################ +# This Dockerfile was generated from the template at distribution/src/docker/Dockerfile +# +# Beginning of multi stage Dockerfile +################################################################################ + +<% /* + This file is passed through Groovy's SimpleTemplateEngine, so dollars and backslashes + have to be escaped in order for them to appear in the final Dockerfile. You + can also comment out blocks, like this one. See: + + https://docs.groovy-lang.org/latest/html/api/groovy/text/SimpleTemplateEngine.html + + We use control-flow tags in this file to conditionally render the content. The + layout/presentation here has been adjusted so that it looks reasonable when rendered, + at the slight expense of how it looks here. + + Note that this file is also filtered to squash together newlines, so we can + add as many newlines here as necessary to improve legibility. +*/ %> + +################################################################################ +# Build stage 1 `builder`: +# Extract Elasticsearch artifact +################################################################################ + +FROM ${base_image} AS builder + +# Install required packages to extract the Elasticsearch distribution +RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %> + +RUN mkdir /usr/share/elasticsearch +WORKDIR /usr/share/elasticsearch + +# Fetch the appropriate Elasticsearch distribution for this architecture. +# Keep this command on one line - it is replaced with a `COPY` during local builds. +# It uses the `arch` shell command to fetch the correct distro for the build machine, +# which is needed for Docker Hub builds. +RUN curl --retry 10 -S -L --output /tmp/elasticsearch.tar.gz https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/elasticsearch-${version}-linux-\$(arch).tar.gz + +RUN tar -zxf /tmp/elasticsearch.tar.gz --strip-components=1 + +# The distribution includes a `config` directory, no need to create it +COPY ${config_dir}/elasticsearch.yml config/ +COPY ${config_dir}/log4j2.properties config/log4j2.docker.properties + +# 1. Configure the distribution for Docker +# 2. Create required directory +# 3. Move the distribution's default logging config aside +# 4. Move the generated docker logging config so that it is the default +# 5. Reset permissions on all directories +# 6. Reset permissions on all files +# 7. Make CLI tools executable +# 8. Make some directories writable. `bin` must be writable because +# plugins can install their own CLI utilities. +# 9. Make some files writable +RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elasticsearch-env && \\ + mkdir data && \\ + mv config/log4j2.properties config/log4j2.file.properties && \\ + mv config/log4j2.docker.properties config/log4j2.properties && \\ + find . -type d -exec chmod 0555 {} + && \\ + find . -type f -exec chmod 0444 {} + && \\ + chmod 0555 bin/* jdk/bin/* jdk/lib/jspawnhelper modules/x-pack-ml/platform/linux-*/bin/* && \\ + chmod 0775 bin config config/jvm.options.d data logs plugins && \\ + find config -type f -exec chmod 0664 {} + + +# Add plugins infrastructure +RUN mkdir -p /opt/plugins/archive +RUN chmod -R 0555 /opt/plugins + +RUN mkdir -p /fips/libs +COPY fips/libs/*.jar /fips/libs/ + +COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/ +RUN set -eux ; \\ + for beat in filebeat metricbeat ; do \\ + if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\ + echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\ + exit 1 ; \\ + fi ; \\ + if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\ + echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\ + exit 1 ; \\ + fi ; \\ + mkdir -p /opt/\$beat ; \\ + tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\ + done + +COPY plugins/*.zip /opt/plugins/archive/ + +RUN chown 1000:1000 /opt/plugins/archive/* +RUN chmod 0444 /opt/plugins/archive/* + +COPY fips/resources/fips_java.security /usr/share/elasticsearch/config/fips_java.security +COPY fips/resources/fips_java.policy /usr/share/elasticsearch/config/fips_java.policy + +WORKDIR /usr/share/elasticsearch/config + +## Add fips specific JVM options +RUN cat < /usr/share/elasticsearch/config/jvm.options.d/fips.options +-Djavax.net.ssl.keyStoreType=BCFKS +-Dorg.bouncycastle.fips.approved_only=true +-Djava.security.properties=config/fips_java.security +-Djava.security.policy=config/fips_java.policy +EOF + + +################################################################################ +# Build stage 2 (the actual Elasticsearch image): +# +# Copy elasticsearch from stage 1 +# Add entrypoint +################################################################################ + +FROM ${base_image} + +RUN <%= retry.loop(package_manager, + "export DEBIAN_FRONTEND=noninteractive && \n" + + " ${package_manager} update && \n" + + " ${package_manager} upgrade && \n" + + " ${package_manager} add --no-cache \n" + + " bash java-cacerts curl libstdc++ libsystemd netcat-openbsd p11-kit p11-kit-trust posix-libc-utils shadow tini unzip zip zstd && \n" + + " rm -rf /var/cache/apk/* " + ) %> + +# Set Bash as the default shell for future commands +SHELL ["/bin/bash", "-c"] + +RUN groupadd -g 1000 elasticsearch && \ + adduser -G elasticsearch -u 1000 elasticsearch -D --home /usr/share/elasticsearch elasticsearch && \ + adduser elasticsearch root && \ + chown -R 0:0 /usr/share/elasticsearch + +ENV ELASTIC_CONTAINER=true + +WORKDIR /usr/share/elasticsearch + +COPY --from=builder --chown=0:0 /usr/share/elasticsearch /usr/share/elasticsearch +ENV PATH=/usr/share/elasticsearch/bin:\$PATH +ENV SHELL=/bin/bash +COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh + +# 1. Sync the user and group permissions of /etc/passwd +# 2. Set correct permissions of the entrypoint +# 3. Ensure that there are no files with setuid or setgid, in order to mitigate "stackclash" attacks. +# We've already run this in previous layers so it ought to be a no-op. +# 4. Replace OpenJDK's built-in CA certificate keystore with the one from the OS +# vendor. The latter is superior in several ways. +# REF: https://github.com/elastic/elasticsearch-docker/issues/171 +# 5. Tighten up permissions on the ES home dir (the permissions of the contents are handled earlier) +# 6. You can't install plugins that include configuration when running as `elasticsearch` and the `config` +# dir is owned by `root`, because the installed tries to manipulate the permissions on the plugin's +# config directory. +RUN chmod g=u /etc/passwd && \\ + chmod 0555 /usr/local/bin/docker-entrypoint.sh && \\ + find / -xdev -perm -4000 -exec chmod ug-s {} + && \\ + chmod 0775 /usr/share/elasticsearch && \\ + chown elasticsearch bin config config/jvm.options.d data logs plugins + +RUN ln -sf /etc/ssl/certs/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts + +EXPOSE 9200 9300 + +LABEL org.label-schema.build-date="${build_date}" \\ + org.label-schema.license="${license}" \\ + org.label-schema.name="Elasticsearch" \\ + org.label-schema.schema-version="1.0" \\ + org.label-schema.url="https://www.elastic.co/products/elasticsearch" \\ + org.label-schema.usage="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \\ + org.label-schema.vcs-ref="${git_revision}" \\ + org.label-schema.vcs-url="https://github.com/elastic/elasticsearch" \\ + org.label-schema.vendor="Elastic" \\ + org.label-schema.version="${version}" \\ + org.opencontainers.image.created="${build_date}" \\ + org.opencontainers.image.documentation="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \\ + org.opencontainers.image.licenses="${license}" \\ + org.opencontainers.image.revision="${git_revision}" \\ + org.opencontainers.image.source="https://github.com/elastic/elasticsearch" \\ + org.opencontainers.image.title="Elasticsearch" \\ + org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \\ + org.opencontainers.image.vendor="Elastic" \\ + org.opencontainers.image.version="${version}" + +LABEL name="Elasticsearch" \\ + maintainer="infra@elastic.co" \\ + vendor="Elastic" \\ + version="${version}" \\ + release="1" \\ + summary="Elasticsearch" \\ + description="You know, for search." + +RUN mkdir /licenses && ln LICENSE.txt /licenses/LICENSE + +ENTRYPOINT ["/sbin/tini", "--"] +CMD ["/app/elasticsearch.sh"] + +USER 1000:0 + +COPY --from=builder --chown=0:0 /opt /opt +ENV ES_PLUGIN_ARCHIVE_DIR=/opt/plugins/archive +WORKDIR /usr/share/elasticsearch +COPY --from=builder --chown=0:0 /fips/libs/*.jar /usr/share/elasticsearch/lib/ + +################################################################################ +# End of multi-stage Dockerfile +################################################################################