Merge remote-tracking branch 'origin/main' into mp/snapshots-in-progress-migration

Author: ywangd

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -22,10 +22,12 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:15a4191ff8ec8305dcba449365e8a1586c9cda8e016ae838d960b9009c6a5cac",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:c4e10ecf3d8a21cf4be2fb53a2f522de50e14c80ce1da487e3ffd13f4d48d24d",
         "-wolfi",
         "apk"
     ),
+
+    FIPS("docker.elastic.co/wolfi/chainguard-base-fips:sha256-feb7aeb1bbcb331afa089388f2fa1e81997fc24642ca4fa06b7e502ff599a4cf", "-fips", "apk"),
     // spotless:on
     // Based on WOLFI above, with more extras. We don't set a base image because
     // we programmatically extend from the wolfi image.

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -29,38 +29,13 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         ListMultimap<Class<?>, String> map = ArrayListMultimap.create(1, 200);
         map.put(LegacyRestTestBasePlugin.class, ":docs");
         map.put(LegacyRestTestBasePlugin.class, ":distribution:docker");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:lang-expression");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:lang-mustache");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:mapper-extras");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:parent-join");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:percolator");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:rank-eval");
         map.put(LegacyRestTestBasePlugin.class, ":modules:reindex");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:repository-url");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:transport-netty4");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-icu");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-kuromoji");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-nori");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-phonetic");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-smartcn");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-stempel");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-ukrainian");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-azure-classic");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-ec2");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-gce");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:mapper-annotated-text");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:store-smb");
         map.put(LegacyRestTestBasePlugin.class, ":qa:ccs-rolling-upgrade-remote-cluster");
         map.put(LegacyRestTestBasePlugin.class, ":qa:mixed-cluster");
         map.put(LegacyRestTestBasePlugin.class, ":qa:multi-cluster-search");
         map.put(LegacyRestTestBasePlugin.class, ":qa:remote-clusters");
         map.put(LegacyRestTestBasePlugin.class, ":qa:repository-multi-version");
         map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade-legacy");
-        map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-http");
-        map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-disabled");
-        map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-with-all-dependencies");
-        map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-plugins");
-        map.put(LegacyRestTestBasePlugin.class, ":qa:system-indices");
         map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-apm-integration");
         map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-delayed-aggs");
         map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-die-with-dignity");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerFipsElasticsearchDistributionType.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.distribution;
+
+import org.elasticsearch.gradle.ElasticsearchDistributionType;
+
+public class DockerFipsElasticsearchDistributionType implements ElasticsearchDistributionType {
+
+    DockerFipsElasticsearchDistributionType() {}
+
+    @Override
+    public String getName() {
+        return "dockerFips";
+    }
+
+    @Override
+    public boolean isDocker() {
+        return true;
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java

@@ -14,19 +14,22 @@
 import java.util.List;
 
 public class InternalElasticsearchDistributionTypes {
+
     public static final ElasticsearchDistributionType DEB = new DebElasticsearchDistributionType();
     public static final ElasticsearchDistributionType RPM = new RpmElasticsearchDistributionType();
     public static final ElasticsearchDistributionType DOCKER = new DockerElasticsearchDistributionType();
     public static final ElasticsearchDistributionType DOCKER_IRONBANK = new DockerIronBankElasticsearchDistributionType();
     public static final ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType();
     public static final ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType DOCKER_FIPS = new DockerFipsElasticsearchDistributionType();
 
     public static final List<ElasticsearchDistributionType> ALL_INTERNAL = List.of(
         DEB,
         RPM,
         DOCKER,
         DOCKER_IRONBANK,
+        DOCKER_WOLFI,
         DOCKER_CLOUD_ESS,
-        DOCKER_WOLFI
+        DOCKER_FIPS
     );
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java

@@ -51,6 +51,7 @@
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DEB;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_CLOUD_ESS;
+import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_FIPS;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_IRONBANK;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.RPM;
@@ -151,6 +152,7 @@ private static Map<ElasticsearchDistributionType, TaskProvider<?>> lifecycleTask
         lifecyleTasks.put(DOCKER_IRONBANK, project.getTasks().register(taskPrefix + ".docker-ironbank"));
         lifecyleTasks.put(DOCKER_CLOUD_ESS, project.getTasks().register(taskPrefix + ".docker-cloud-ess"));
         lifecyleTasks.put(DOCKER_WOLFI, project.getTasks().register(taskPrefix + ".docker-wolfi"));
+        lifecyleTasks.put(DOCKER_FIPS, project.getTasks().register(taskPrefix + ".docker-fips"));
         lifecyleTasks.put(ARCHIVE, project.getTasks().register(taskPrefix + ".archives"));
         lifecyleTasks.put(DEB, project.getTasks().register(taskPrefix + ".packages"));
         lifecyleTasks.put(RPM, lifecyleTasks.get(DEB));

build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java

@@ -56,6 +56,7 @@ public class BasePluginBuildPlugin implements Plugin<Project> {
     public static final String BUNDLE_PLUGIN_TASK_NAME = "bundlePlugin";
     public static final String EXPLODED_BUNDLE_PLUGIN_TASK_NAME = "explodedBundlePlugin";
     public static final String EXPLODED_BUNDLE_CONFIG = "explodedBundleZip";
+    public static final Attribute<Boolean> EXPLODED_PLUGIN_BUNDLE_ATTRIBUTE = Attribute.of("exploded-plugin-bundle", Boolean.class);
 
     protected final ProviderFactory providerFactory;
 
@@ -170,6 +171,7 @@ private TaskProvider<Zip> createBundleTasks(final Project project, PluginPropert
         explodedBundleZip.setCanBeResolved(false);
         explodedBundleZip.setCanBeConsumed(true);
         explodedBundleZip.getAttributes().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE);
+        explodedBundleZip.getAttributes().attribute(EXPLODED_PLUGIN_BUNDLE_ATTRIBUTE, true);
         project.getArtifacts().add(EXPLODED_BUNDLE_CONFIG, explodedBundle);
         return bundle;
     }

build.gradle

@@ -295,7 +295,16 @@ allprojects {
   tasks.register('resolveAllDependencies', ResolveAllDependencies) {
     def ignoredPrefixes = [DistributionDownloadPlugin.ES_DISTRO_CONFIG_PREFIX, "jdbcDriver"]
     configs = project.configurations.matching { config -> ignoredPrefixes.any { config.name.startsWith(it) } == false }
-    resolveJavaToolChain = true
+    if(project.path == ':') {
+      resolveJavaToolChain = true
+
+      // ensure we have best possible caching of bwc builds
+      dependsOn ":distribution:bwc:bugfix:buildBwcLinuxTar"
+      dependsOn ":distribution:bwc:bugfix2:buildBwcLinuxTar"
+      dependsOn ":distribution:bwc:minor:buildBwcLinuxTar"
+      dependsOn ":distribution:bwc:staged:buildBwcLinuxTar"
+      dependsOn ":distribution:bwc:staged2:buildBwcLinuxTar"
+    }
     if (project.path.contains("fixture")) {
       dependsOn tasks.withType(ComposePull)
     }
@@ -306,6 +315,7 @@ allprojects {
       // ensure we resolve p2 dependencies for the spotless eclipse formatter
       dependsOn "spotlessJavaCheck"
     }
+
   }
 
   ext.withReleaseBuild = { Closure config ->

distribution/docker/build.gradle

@@ -2,6 +2,7 @@ import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.DockerBase
 import org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes
+import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.internal.docker.DockerBuildTask
 import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin
 import org.elasticsearch.gradle.internal.docker.DockerSupportService
@@ -17,6 +18,8 @@ apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 apply plugin: 'elasticsearch.internal-distribution-download'
 apply plugin: 'elasticsearch.dra-artifacts'
+apply plugin: 'elasticsearch.jdk-download'
+apply plugin: 'elasticsearch.repositories'
 
 String buildId = providers.systemProperty('build.id').getOrNull()
 boolean useLocalArtifacts = buildId != null && buildId.isBlank() == false && useDra == false
@@ -93,6 +96,7 @@ configurations {
   filebeat_x86_64
   metricbeat_aarch64
   metricbeat_x86_64
+  fips
 }
 
 String tiniArch = Architecture.current() == Architecture.AARCH64 ? 'arm64' : 'amd64'
@@ -109,6 +113,8 @@ dependencies {
   filebeat_x86_64 "beats:filebeat:${VersionProperties.elasticsearch}:[email protected]"
   metricbeat_aarch64 "beats:metricbeat:${VersionProperties.elasticsearch}:[email protected]"
   metricbeat_x86_64 "beats:metricbeat:${VersionProperties.elasticsearch}:[email protected]"
+  fips "org.bouncycastle:bc-fips:1.0.2.5"
+  fips "org.bouncycastle:bctls-fips:1.0.19"
 }
 
 ext.expansions = { Architecture architecture, DockerBase base ->
@@ -286,6 +292,34 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
           filter TransformLog4jConfigFilter
         }
       }
+      if(base == DockerBase.FIPS) {
+
+        // If we're performing a release build, but `build.id` hasn't been set, we can
+        // infer that we're not at the Docker building stage of the build, and therefore
+        // we should skip the beats part of the build.
+        String buildId = providers.systemProperty('build.id').getOrNull()
+        boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
+
+        if (includeBeats) {
+          from configurations.getByName("filebeat_${architecture.classifier}")
+          from configurations.getByName("metricbeat_${architecture.classifier}")
+          // For some reason, the artifact name can differ depending on what repository we used.
+          rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
+        }
+
+        into("plugins") {
+          from configurations.allPlugins
+        }
+
+        into("fips") {
+          into("libs") {
+            from configurations.fips
+          }
+          into("resources") {
+            from tasks.named('fipsResources')
+          }
+        }
+      }
 
       Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(
         project.gradle.sharedServices,
@@ -431,7 +465,7 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
   }
 }
 
-void addBuildEssDockerImageTask(Architecture architecture) {
+void addBuildCloudDockerImageTasks(Architecture architecture) {
   DockerBase dockerBase = DockerBase.CLOUD_ESS
   String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
   String contextDir = "${project.buildDir}/docker-context/elasticsearch${dockerBase.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}"
@@ -463,10 +497,10 @@ void addBuildEssDockerImageTask(Architecture architecture) {
       from(projectDir.resolve("src/docker/Dockerfile.ess")) {
         expand(
           [
-            base_image: "elasticsearch${baseSuffix}:${architecture.classifier}",
+            base_image : "elasticsearch${baseSuffix}:${architecture.classifier}",
             docker_base: "${dockerBase.name().toLowerCase()}",
-            version: "${VersionProperties.elasticsearch}",
-            retry: ShellRetry
+            version    : "${VersionProperties.elasticsearch}",
+            retry      : ShellRetry
           ]
         )
         filter SquashNewlinesFilter
@@ -501,17 +535,24 @@ void addBuildEssDockerImageTask(Architecture architecture) {
   }
 }
 
+// fips
+TaskProvider<ExportElasticsearchBuildResourcesTask> fipsResourcesTask = tasks.register('fipsResources', ExportElasticsearchBuildResourcesTask)
+fipsResourcesTask.configure {
+  outputDir = project.layout.buildDirectory.dir('fips-resources').get().asFile
+  copy 'fips_java.security'
+  copy 'fips_java.policy'
+}
+
 for (final Architecture architecture : Architecture.values()) {
   for (final DockerBase base : DockerBase.values()) {
     if (base == DockerBase.CLOUD_ESS) {
-      continue
+      addBuildCloudDockerImageTasks(architecture)
+    } else {
+      addBuildDockerContextTask(architecture, base)
+      addTransformDockerContextTask(architecture, base)
+      addBuildDockerImageTask(architecture, base)
     }
-    addBuildDockerContextTask(architecture, base)
-    addTransformDockerContextTask(architecture, base)
-    addBuildDockerImageTask(architecture, base)
   }
-
-  addBuildEssDockerImageTask(architecture)
 }
 
 def exportDockerImages = tasks.register("exportDockerImages")
@@ -533,14 +574,17 @@ subprojects { Project subProject ->
       base = DockerBase.CLOUD_ESS
     } else if (subProject.name.contains('wolfi-')) {
       base = DockerBase.WOLFI
+    } else if (subProject.name.contains('fips-')) {
+      base = DockerBase.FIPS
     }
 
     final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
     final String extension =
       (base == DockerBase.IRON_BANK ? 'ironbank.tar' :
         (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' :
-          (base == DockerBase.WOLFI ? 'wolfi.tar' :
-            'docker.tar')))
+          (base == DockerBase.FIPS ? 'fips.tar' :
+            (base == DockerBase.WOLFI ? 'wolfi.tar' :
+              'docker.tar'))))
     final String artifactName = "elasticsearch${arch}${base.suffix}_test"
 
     final String exportTaskName = taskName("export", architecture, base, 'DockerImage')