Merge branch 'main' into layout-build-dedup

Author: fang-xing-esql

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -26,6 +26,8 @@ public enum DockerBase {
         "-wolfi",
         "apk"
     ),
+
+    FIPS("docker.elastic.co/wolfi/chainguard-base-fips:sha256-feb7aeb1bbcb331afa089388f2fa1e81997fc24642ca4fa06b7e502ff599a4cf", "-fips", "apk"),
     // spotless:on
     // Based on WOLFI above, with more extras. We don't set a base image because
     // we programmatically extend from the wolfi image.

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -29,15 +29,7 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         ListMultimap<Class<?>, String> map = ArrayListMultimap.create(1, 200);
         map.put(LegacyRestTestBasePlugin.class, ":docs");
         map.put(LegacyRestTestBasePlugin.class, ":distribution:docker");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:lang-expression");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:lang-mustache");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:mapper-extras");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:parent-join");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:percolator");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:rank-eval");
         map.put(LegacyRestTestBasePlugin.class, ":modules:reindex");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:repository-url");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:transport-netty4");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-icu");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-kuromoji");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-nori");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerFipsElasticsearchDistributionType.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.distribution;
+
+import org.elasticsearch.gradle.ElasticsearchDistributionType;
+
+public class DockerFipsElasticsearchDistributionType implements ElasticsearchDistributionType {
+
+    DockerFipsElasticsearchDistributionType() {}
+
+    @Override
+    public String getName() {
+        return "dockerFips";
+    }
+
+    @Override
+    public boolean isDocker() {
+        return true;
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java

@@ -14,19 +14,22 @@
 import java.util.List;
 
 public class InternalElasticsearchDistributionTypes {
+
     public static final ElasticsearchDistributionType DEB = new DebElasticsearchDistributionType();
     public static final ElasticsearchDistributionType RPM = new RpmElasticsearchDistributionType();
     public static final ElasticsearchDistributionType DOCKER = new DockerElasticsearchDistributionType();
     public static final ElasticsearchDistributionType DOCKER_IRONBANK = new DockerIronBankElasticsearchDistributionType();
     public static final ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType();
     public static final ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType DOCKER_FIPS = new DockerFipsElasticsearchDistributionType();
 
     public static final List<ElasticsearchDistributionType> ALL_INTERNAL = List.of(
         DEB,
         RPM,
         DOCKER,
         DOCKER_IRONBANK,
+        DOCKER_WOLFI,
         DOCKER_CLOUD_ESS,
-        DOCKER_WOLFI
+        DOCKER_FIPS
     );
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java

@@ -51,6 +51,7 @@
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DEB;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_CLOUD_ESS;
+import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_FIPS;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_IRONBANK;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.RPM;
@@ -151,6 +152,7 @@ private static Map<ElasticsearchDistributionType, TaskProvider<?>> lifecycleTask
         lifecyleTasks.put(DOCKER_IRONBANK, project.getTasks().register(taskPrefix + ".docker-ironbank"));
         lifecyleTasks.put(DOCKER_CLOUD_ESS, project.getTasks().register(taskPrefix + ".docker-cloud-ess"));
         lifecyleTasks.put(DOCKER_WOLFI, project.getTasks().register(taskPrefix + ".docker-wolfi"));
+        lifecyleTasks.put(DOCKER_FIPS, project.getTasks().register(taskPrefix + ".docker-fips"));
         lifecyleTasks.put(ARCHIVE, project.getTasks().register(taskPrefix + ".archives"));
         lifecyleTasks.put(DEB, project.getTasks().register(taskPrefix + ".packages"));
         lifecyleTasks.put(RPM, lifecyleTasks.get(DEB));

build.gradle

@@ -295,7 +295,16 @@ allprojects {
   tasks.register('resolveAllDependencies', ResolveAllDependencies) {
     def ignoredPrefixes = [DistributionDownloadPlugin.ES_DISTRO_CONFIG_PREFIX, "jdbcDriver"]
     configs = project.configurations.matching { config -> ignoredPrefixes.any { config.name.startsWith(it) } == false }
-    resolveJavaToolChain = true
+    if(project.path == ':') {
+      resolveJavaToolChain = true
+
+      // ensure we have best possible caching of bwc builds
+      dependsOn ":distribution:bwc:bugfix:buildBwcLinuxTar"
+      dependsOn ":distribution:bwc:bugfix2:buildBwcLinuxTar"
+      dependsOn ":distribution:bwc:minor:buildBwcLinuxTar"
+      dependsOn ":distribution:bwc:staged:buildBwcLinuxTar"
+      dependsOn ":distribution:bwc:staged2:buildBwcLinuxTar"
+    }
     if (project.path.contains("fixture")) {
       dependsOn tasks.withType(ComposePull)
     }
@@ -306,6 +315,7 @@ allprojects {
       // ensure we resolve p2 dependencies for the spotless eclipse formatter
       dependsOn "spotlessJavaCheck"
     }
+
   }
 
   ext.withReleaseBuild = { Closure config ->

distribution/docker/build.gradle

@@ -2,6 +2,7 @@ import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.DockerBase
 import org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes
+import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.internal.docker.DockerBuildTask
 import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin
 import org.elasticsearch.gradle.internal.docker.DockerSupportService
@@ -17,6 +18,8 @@ apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 apply plugin: 'elasticsearch.internal-distribution-download'
 apply plugin: 'elasticsearch.dra-artifacts'
+apply plugin: 'elasticsearch.jdk-download'
+apply plugin: 'elasticsearch.repositories'
 
 String buildId = providers.systemProperty('build.id').getOrNull()
 boolean useLocalArtifacts = buildId != null && buildId.isBlank() == false && useDra == false
@@ -93,6 +96,7 @@ configurations {
   filebeat_x86_64
   metricbeat_aarch64
   metricbeat_x86_64
+  fips
 }
 
 String tiniArch = Architecture.current() == Architecture.AARCH64 ? 'arm64' : 'amd64'
@@ -109,6 +113,8 @@ dependencies {
   filebeat_x86_64 "beats:filebeat:${VersionProperties.elasticsearch}:[email protected]"
   metricbeat_aarch64 "beats:metricbeat:${VersionProperties.elasticsearch}:[email protected]"
   metricbeat_x86_64 "beats:metricbeat:${VersionProperties.elasticsearch}:[email protected]"
+  fips "org.bouncycastle:bc-fips:1.0.2.5"
+  fips "org.bouncycastle:bctls-fips:1.0.19"
 }
 
 ext.expansions = { Architecture architecture, DockerBase base ->
@@ -286,6 +292,34 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
           filter TransformLog4jConfigFilter
         }
       }
+      if(base == DockerBase.FIPS) {
+
+        // If we're performing a release build, but `build.id` hasn't been set, we can
+        // infer that we're not at the Docker building stage of the build, and therefore
+        // we should skip the beats part of the build.
+        String buildId = providers.systemProperty('build.id').getOrNull()
+        boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
+
+        if (includeBeats) {
+          from configurations.getByName("filebeat_${architecture.classifier}")
+          from configurations.getByName("metricbeat_${architecture.classifier}")
+          // For some reason, the artifact name can differ depending on what repository we used.
+          rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
+        }
+
+        into("plugins") {
+          from configurations.allPlugins
+        }
+
+        into("fips") {
+          into("libs") {
+            from configurations.fips
+          }
+          into("resources") {
+            from tasks.named('fipsResources')
+          }
+        }
+      }
 
       Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(
         project.gradle.sharedServices,
@@ -431,7 +465,7 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
   }
 }
 
-void addBuildEssDockerImageTask(Architecture architecture) {
+void addBuildCloudDockerImageTasks(Architecture architecture) {
   DockerBase dockerBase = DockerBase.CLOUD_ESS
   String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
   String contextDir = "${project.buildDir}/docker-context/elasticsearch${dockerBase.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}"
@@ -463,10 +497,10 @@ void addBuildEssDockerImageTask(Architecture architecture) {
       from(projectDir.resolve("src/docker/Dockerfile.ess")) {
         expand(
           [
-            base_image: "elasticsearch${baseSuffix}:${architecture.classifier}",
+            base_image : "elasticsearch${baseSuffix}:${architecture.classifier}",
             docker_base: "${dockerBase.name().toLowerCase()}",
-            version: "${VersionProperties.elasticsearch}",
-            retry: ShellRetry
+            version    : "${VersionProperties.elasticsearch}",
+            retry      : ShellRetry
           ]
         )
         filter SquashNewlinesFilter
@@ -501,17 +535,24 @@ void addBuildEssDockerImageTask(Architecture architecture) {
   }
 }
 
+// fips
+TaskProvider<ExportElasticsearchBuildResourcesTask> fipsResourcesTask = tasks.register('fipsResources', ExportElasticsearchBuildResourcesTask)
+fipsResourcesTask.configure {
+  outputDir = project.layout.buildDirectory.dir('fips-resources').get().asFile
+  copy 'fips_java.security'
+  copy 'fips_java.policy'
+}
+
 for (final Architecture architecture : Architecture.values()) {
   for (final DockerBase base : DockerBase.values()) {
     if (base == DockerBase.CLOUD_ESS) {
-      continue
+      addBuildCloudDockerImageTasks(architecture)
+    } else {
+      addBuildDockerContextTask(architecture, base)
+      addTransformDockerContextTask(architecture, base)
+      addBuildDockerImageTask(architecture, base)
     }
-    addBuildDockerContextTask(architecture, base)
-    addTransformDockerContextTask(architecture, base)
-    addBuildDockerImageTask(architecture, base)
   }
-
-  addBuildEssDockerImageTask(architecture)
 }
 
 def exportDockerImages = tasks.register("exportDockerImages")
@@ -533,14 +574,17 @@ subprojects { Project subProject ->
       base = DockerBase.CLOUD_ESS
     } else if (subProject.name.contains('wolfi-')) {
       base = DockerBase.WOLFI
+    } else if (subProject.name.contains('fips-')) {
+      base = DockerBase.FIPS
     }
 
     final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
     final String extension =
       (base == DockerBase.IRON_BANK ? 'ironbank.tar' :
         (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' :
-          (base == DockerBase.WOLFI ? 'wolfi.tar' :
-            'docker.tar')))
+          (base == DockerBase.FIPS ? 'fips.tar' :
+            (base == DockerBase.WOLFI ? 'wolfi.tar' :
+              'docker.tar'))))
     final String artifactName = "elasticsearch${arch}${base.suffix}_test"
 
     final String exportTaskName = taskName("export", architecture, base, 'DockerImage')

modules/repository-gcs/licenses/grpc-context-NOTICE.txt renamed to distribution/docker/fips-docker-export/build.gradle

@@ -41,7 +41,7 @@ RUN chmod 0555 /bin/tini
 <% } else { %>
 
 # Install required packages to extract the Elasticsearch distribution
-<% if (docker_base == "wolfi") { %>
+<% if (docker_base == "wolfi" || docker_base == "fips") { %>
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
 <% } else { %>
 RUN <%= retry.loop(package_manager, "${package_manager} install -y findutils tar gzip") %>
@@ -115,6 +115,51 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
     chmod 0775 bin config config/jvm.options.d data logs plugins && \\
     find config -type f -exec chmod 0664 {} +
 
+<% if (docker_base == "fips") { %>
+
+# Add plugins infrastructure
+RUN mkdir -p /opt/plugins/archive
+RUN chmod -R 0555 /opt/plugins
+
+RUN mkdir -p /fips/libs
+COPY fips/libs/*.jar /fips/libs/
+
+COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/
+RUN set -eux ; \\
+    for beat in filebeat metricbeat ; do \\
+      if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\
+        echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\
+        exit 1 ; \\
+      fi ; \\
+      if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\
+        echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\
+        exit 1 ; \\
+      fi ; \\
+      mkdir -p /opt/\$beat ; \\
+      tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\
+    done
+
+COPY plugins/*.zip /opt/plugins/archive/
+
+RUN chown 1000:1000 /opt/plugins/archive/*
+RUN chmod 0444 /opt/plugins/archive/*
+
+COPY fips/resources/fips_java.security /usr/share/elasticsearch/config/fips_java.security
+COPY fips/resources/fips_java.policy /usr/share/elasticsearch/config/fips_java.policy
+
+WORKDIR /usr/share/elasticsearch/config
+
+## Add fips specific JVM options
+RUN cat <<EOF > /usr/share/elasticsearch/config/jvm.options.d/fips.options
+-Djavax.net.ssl.keyStoreType=BCFKS
+-Dorg.bouncycastle.fips.approved_only=true
+-Djava.security.properties=config/fips_java.security
+-Djava.security.policy=config/fips_java.policy
+EOF
+
+<% } %>
+
+
 ################################################################################
 # Build stage 2 (the actual Elasticsearch image):
 #
@@ -134,7 +179,7 @@ RUN ${package_manager} update --setopt=tsflags=nodocs -y && \\
       nc shadow-utils zip findutils unzip procps-ng && \\
     ${package_manager} clean all
 
-<% } else if (docker_base == "wolfi") { %>
+<% } else if (docker_base == "wolfi" || docker_base == "fips") { %>
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +
           "      ${package_manager} update && \n" +
@@ -163,7 +208,7 @@ RUN <%= retry.loop(
 <% } %>
 
 
-<% if (docker_base == "wolfi") { %>
+<% if (docker_base == "wolfi" || docker_base == "fips") { %>
 RUN groupadd -g 1000 elasticsearch && \
     adduser -G elasticsearch -u 1000 elasticsearch -D --home /usr/share/elasticsearch elasticsearch && \
     adduser elasticsearch root && \
@@ -179,7 +224,7 @@ ENV ELASTIC_CONTAINER true
 WORKDIR /usr/share/elasticsearch
 
 COPY --from=builder --chown=0:0 /usr/share/elasticsearch /usr/share/elasticsearch
-<% if (docker_base != "wolfi") { %>
+<% if (docker_base != "wolfi" && docker_base != "fips") { %>
 COPY --from=builder --chown=0:0 /bin/tini /bin/tini
 <% } %>
 
@@ -204,7 +249,7 @@ RUN chmod g=u /etc/passwd && \\
     chmod 0775 /usr/share/elasticsearch && \\
     chown elasticsearch bin config config/jvm.options.d data logs plugins
 
-<% if (docker_base == 'wolfi') { %>
+<% if (docker_base == 'wolfi' || docker_base == "fips") { %>
 RUN ln -sf /etc/ssl/certs/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts
 <% } else { %>
 RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts
@@ -247,7 +292,7 @@ RUN mkdir /licenses && cp LICENSE.txt /licenses/LICENSE
 COPY LICENSE /licenses/LICENSE.addendum
 <% } %>
 
-<% if (docker_base == "wolfi") { %>
+<% if (docker_base == "wolfi" || docker_base == "fips") { %>
 # Our actual entrypoint is `tini`, a minimal but functional init program. It
 # calls the entrypoint we provide, while correctly forwarding signals.
 ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"]
@@ -267,6 +312,12 @@ USER 1000:0
 HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 http://localhost:9200 || exit 1
 <% } %>
 
+<% if (docker_base == 'fips') { %>
+COPY --from=builder --chown=0:0 /opt /opt
+ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive
+WORKDIR /usr/share/elasticsearch
+COPY --from=builder --chown=0:0 /fips/libs/*.jar /usr/share/elasticsearch/lib/
+<% } %>
 ################################################################################
 # End of multi-stage Dockerfile
 ################################################################################

distribution/docker/src/docker/Dockerfile

@@ -0,0 +1,5 @@
+pr: 124062
+summary: Upgrade to repository-gcs to use com.google.cloud:google-cloud-storage-bom:2.50.0
+area: Snapshot/Restore
+type: upgrade
+issues: []