Skip to content

Commit 89f8c6d

Browse files
mark-vieiramark-vieira
authored
Allow for building multi-arch docker images via buildx (#89986) (#90046)
1 parent 76b93b0 commit 89f8c6d

File tree

4 files changed

+84
-24
lines changed

4 files changed

+84
-24
lines changed

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
*/
88
package org.elasticsearch.gradle.internal.docker;
99

10+
import org.elasticsearch.gradle.Architecture;
1011
import org.elasticsearch.gradle.LoggedExec;
1112
import org.gradle.api.DefaultTask;
1213
import org.gradle.api.GradleException;
@@ -54,13 +55,15 @@ public class DockerBuildTask extends DefaultTask {
5455
private boolean noCache = true;
5556
private String[] baseImages;
5657
private MapProperty<String, String> buildArgs;
58+
private Property<String> platform;
5759

5860
@Inject
5961
public DockerBuildTask(WorkerExecutor workerExecutor, ObjectFactory objectFactory, ProjectLayout projectLayout) {
6062
this.workerExecutor = workerExecutor;
6163
this.markerFile = objectFactory.fileProperty();
6264
this.dockerContext = objectFactory.directoryProperty();
6365
this.buildArgs = objectFactory.mapProperty(String.class, String.class);
66+
this.platform = objectFactory.property(String.class).convention(Architecture.current().dockerPlatform);
6467
this.markerFile.set(projectLayout.getBuildDirectory().file("markers/" + this.getName() + ".marker"));
6568
}
6669

@@ -74,6 +77,7 @@ public void build() {
7477
params.getNoCache().set(noCache);
7578
params.getBaseImages().set(Arrays.asList(baseImages));
7679
params.getBuildArgs().set(buildArgs);
80+
params.getPlatform().set(platform);
7781
});
7882
}
7983

@@ -124,8 +128,9 @@ public MapProperty<String, String> getBuildArgs() {
124128
return buildArgs;
125129
}
126130

127-
public void setBuildArgs(MapProperty<String, String> buildArgs) {
128-
this.buildArgs = buildArgs;
131+
@Input
132+
public Property<String> getPlatform() {
133+
return platform;
129134
}
130135

131136
@OutputFile
@@ -176,12 +181,21 @@ public void execute() {
176181
}
177182

178183
final List<String> tags = parameters.getTags().get();
184+
final boolean isCrossPlatform = parameters.getPlatform().get().equals(Architecture.current().dockerPlatform) == false;
179185

180186
LoggedExec.exec(execOperations, spec -> {
181187
spec.executable("docker");
182188

189+
if (isCrossPlatform) {
190+
spec.args("buildx");
191+
}
192+
183193
spec.args("build", parameters.getDockerContext().get().getAsFile().getAbsolutePath());
184194

195+
if (isCrossPlatform) {
196+
spec.args("--platform", parameters.getPlatform().get());
197+
}
198+
185199
if (parameters.getNoCache().get()) {
186200
spec.args("--no-cache");
187201
}
@@ -228,5 +242,7 @@ interface Parameters extends WorkParameters {
228242
ListProperty<String> getBaseImages();
229243

230244
MapProperty<String, String> getBuildArgs();
245+
246+
Property<String> getPlatform();
231247
}
232248
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java‎

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
*/
88
package org.elasticsearch.gradle.internal.docker;
99

10+
import org.elasticsearch.gradle.Architecture;
1011
import org.elasticsearch.gradle.Version;
1112
import org.elasticsearch.gradle.internal.info.BuildParams;
1213
import org.gradle.api.GradleException;
@@ -23,26 +24,31 @@
2324
import java.nio.file.Files;
2425
import java.nio.file.Path;
2526
import java.nio.file.Paths;
27+
import java.util.Arrays;
2628
import java.util.Collections;
2729
import java.util.HashMap;
30+
import java.util.HashSet;
2831
import java.util.List;
2932
import java.util.Locale;
3033
import java.util.Map;
3134
import java.util.Optional;
35+
import java.util.Set;
3236
import java.util.stream.Collectors;
3337

3438
import javax.inject.Inject;
3539

40+
import static java.util.function.Predicate.not;
41+
3642
/**
3743
* Build service for detecting available Docker installation and checking for compatibility with Elasticsearch Docker image build
3844
* requirements. This includes a minimum version requirement, as well as the ability to run privileged commands.
3945
*/
4046
public abstract class DockerSupportService implements BuildService<DockerSupportService.Parameters> {
4147

42-
private static Logger LOGGER = Logging.getLogger(DockerSupportService.class);
48+
private static final Logger LOGGER = Logging.getLogger(DockerSupportService.class);
4349
// Defines the possible locations of the Docker CLI. These will be searched in order.
44-
private static String[] DOCKER_BINARIES = { "/usr/bin/docker", "/usr/local/bin/docker" };
45-
private static String[] DOCKER_COMPOSE_BINARIES = { "/usr/local/bin/docker-compose", "/usr/bin/docker-compose" };
50+
private static final String[] DOCKER_BINARIES = { "/usr/bin/docker", "/usr/local/bin/docker" };
51+
private static final String[] DOCKER_COMPOSE_BINARIES = { "/usr/local/bin/docker-compose", "/usr/bin/docker-compose" };
4652
private static final Version MINIMUM_DOCKER_VERSION = Version.fromString("17.05.0");
4753

4854
private final ExecOperations execOperations;
@@ -65,6 +71,7 @@ public DockerAvailability getDockerAvailability() {
6571
Version version = null;
6672
boolean isVersionHighEnough = false;
6773
boolean isComposeAvailable = false;
74+
Set<Architecture> supportedArchitectures = new HashSet<>();
6875

6976
// Check if the Docker binary exists
7077
final Optional<String> dockerBinary = getDockerPath();
@@ -92,6 +99,25 @@ public DockerAvailability getDockerAvailability() {
9299
if (lastResult.isSuccess() && composePath.isPresent()) {
93100
isComposeAvailable = runCommand(composePath.get(), "version").isSuccess();
94101
}
102+
103+
// Now let's check if buildx is available and what supported platforms exist
104+
if (lastResult.isSuccess()) {
105+
Result buildxResult = runCommand(dockerPath, "buildx", "inspect", "--bootstrap");
106+
if (buildxResult.isSuccess()) {
107+
supportedArchitectures = buildxResult.stdout()
108+
.lines()
109+
.filter(l -> l.startsWith("Platforms:"))
110+
.map(l -> l.substring(10))
111+
.flatMap(l -> Arrays.stream(l.split(",")).filter(not(String::isBlank)))
112+
.map(String::trim)
113+
.map(s -> Arrays.stream(Architecture.values()).filter(a -> a.dockerPlatform.equals(s)).findAny())
114+
.filter(Optional::isPresent)
115+
.map(Optional::get)
116+
.collect(Collectors.toSet());
117+
} else {
118+
supportedArchitectures = Set.of(Architecture.current());
119+
}
120+
}
95121
}
96122
}
97123
}
@@ -104,7 +130,8 @@ public DockerAvailability getDockerAvailability() {
104130
isVersionHighEnough,
105131
dockerPath,
106132
version,
107-
lastResult
133+
lastResult,
134+
supportedArchitectures
108135
);
109136
}
110137

@@ -334,7 +361,10 @@ public record DockerAvailability(
334361
Version version,
335362

336363
// Information about the last command executes while probing Docker, or null.
337-
Result lastCommand
364+
Result lastCommand,
365+
366+
// Supported build architectures
367+
Set<Architecture> supportedArchitectures
338368
) {}
339369

340370
/**

‎build-tools/src/main/java/org/elasticsearch/gradle/Architecture.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,15 @@
1010

1111
public enum Architecture {
1212

13-
X64("x86_64"),
14-
AARCH64("aarch64");
13+
X64("x86_64", "linux/amd64"),
14+
AARCH64("aarch64", "linux/arm64");
1515

1616
public final String classifier;
17+
public final String dockerPlatform;
1718

18-
Architecture(String classifier) {
19+
Architecture(String classifier, String dockerPlatform) {
1920
this.classifier = classifier;
21+
this.dockerPlatform = dockerPlatform;
2022
}
2123

2224
public static Architecture current() {

‎distribution/docker/build.gradle‎

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,12 @@ import org.elasticsearch.gradle.VersionProperties
44
import org.elasticsearch.gradle.internal.DockerBase
55
import org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes
66
import org.elasticsearch.gradle.internal.docker.DockerBuildTask
7+
import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin
8+
import org.elasticsearch.gradle.internal.docker.DockerSupportService
79
import org.elasticsearch.gradle.internal.docker.ShellRetry
810
import org.elasticsearch.gradle.internal.docker.TransformLog4jConfigFilter
911
import org.elasticsearch.gradle.internal.info.BuildParams
12+
import org.elasticsearch.gradle.util.GradleUtils
1013

1114
import java.nio.file.Path
1215
import java.time.temporal.ChronoUnit
@@ -271,7 +274,7 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
271274
rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
272275
}
273276

274-
onlyIf { Architecture.current() == architecture }
277+
onlyIf { isArchitectureSupported(architecture) }
275278
}
276279

277280
if (base == DockerBase.IRON_BANK) {
@@ -318,12 +321,12 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
318321
inputs.property(k, { v.toString() })
319322
}
320323

321-
onlyIf { Architecture.current() == architecture }
324+
onlyIf { isArchitectureSupported(architecture) }
322325
}
323326
}
324327

325328

326-
private static List<String> generateTags(DockerBase base) {
329+
private static List<String> generateTags(DockerBase base, Architecture architecture) {
327330
final String version = VersionProperties.elasticsearch
328331

329332
String image = "elasticsearch${base.suffix}"
@@ -333,11 +336,13 @@ private static List<String> generateTags(DockerBase base) {
333336
namespace += '-ci'
334337
}
335338

336-
return [
337-
"${image}:test",
338-
"${image}:${version}",
339-
"docker.elastic.co/${namespace}/${image}:${version}"
340-
]
339+
def tags = ["${image}:${architecture.classifier}"]
340+
341+
if (architecture == Architecture.current()) {
342+
tags.addAll(["${image}:test", "${image}:${version}", "docker.elastic.co/${namespace}/${image}:${version}"])
343+
}
344+
345+
return tags
341346
}
342347

343348
void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
@@ -350,7 +355,8 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
350355
dockerContext.fileProvider(transformTask.map { Sync task -> task.getDestinationDir() })
351356

352357
noCache = BuildParams.isCi
353-
tags = generateTags(base)
358+
tags = generateTags(base, architecture)
359+
platform = architecture.dockerPlatform
354360

355361
// We don't build the Iron Bank image when we release Elasticsearch, as there's
356362
// separate process for submitting new releases. However, for testing we do a
@@ -375,7 +381,7 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
375381
baseImages = [base.image]
376382
}
377383

378-
onlyIf { Architecture.current() == architecture }
384+
onlyIf { isArchitectureSupported(architecture) }
379385
}
380386

381387
if (base != DockerBase.IRON_BANK && base != DockerBase.CLOUD && base != DockerBase.CLOUD_ESS) {
@@ -419,9 +425,10 @@ void addBuildEssDockerImageTask(Architecture architecture) {
419425

420426
noCache = BuildParams.isCi
421427
baseImages = []
422-
tags = generateTags(base)
428+
tags = generateTags(base, architecture)
429+
platform = architecture.dockerPlatform
423430

424-
onlyIf { Architecture.current() == architecture }
431+
onlyIf { isArchitectureSupported(architecture) }
425432
}
426433

427434
tasks.named("assemble").configure {
@@ -442,6 +449,11 @@ for (final Architecture architecture : Architecture.values()) {
442449
addBuildEssDockerImageTask(architecture)
443450
}
444451

452+
boolean isArchitectureSupported(Architecture architecture) {
453+
Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
454+
return serviceProvider.get().dockerAvailability.supportedArchitectures().contains(architecture)
455+
}
456+
445457
/*
446458
* The export subprojects write out the generated Docker images to disk, so
447459
* that they can be easily reloaded, for example into a VM for distribution testing
@@ -481,10 +493,10 @@ subprojects { Project subProject ->
481493
args "save",
482494
"-o",
483495
tarFile,
484-
"elasticsearch${base.suffix}:test"
496+
"elasticsearch${base.suffix}:${architecture.classifier}"
485497

486498
dependsOn(parent.path + ":" + buildTaskName)
487-
onlyIf { Architecture.current() == architecture }
499+
onlyIf { isArchitectureSupported(architecture) }
488500
}
489501

490502
artifacts.add('default', file(tarFile)) {

0 commit comments

Comments
 (0)