Skip to content

Commit 67f2453

Browse files
dlvenabledlvenable
authored
Support building and releasing Docker multi-architecture images (opensearch-project#6411)
Support building Docker multi-architecture images and releasing these in the GitHub Actions release project. Continues to build the local architecture with the existing docker release task. Resolves opensearch-project#6405, opensearch-project#6410. Also stops using the Palatir Docker plugin and uses Docker buildx directly. Resolves opensearch-project#5313. Signed-off-by: David Venable <dlv@amazon.com>
1 parent eed7755 commit 67f2453

File tree

3 files changed

+125
-27
lines changed

3 files changed

+125
-27
lines changed

‎.github/workflows/release.yml‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,6 @@ jobs:
4747
- name: Build Maven Artifacts
4848
run: ./gradlew publishAllPublicationsToMavenRepository
4949

50-
- name: Build Docker Image
51-
run: ./gradlew :release:docker:docker
52-
5350
- name: Upload Archives to Archives Bucket
5451
run: ./gradlew :release:archives:uploadArchives -Pregion=us-east-1 -Pbucket=${{ secrets.ARCHIVES_BUCKET_NAME }} -Pprofile=default -PbuildNumber=${{ github.run_number }}
5552

@@ -63,10 +60,8 @@ jobs:
6360
registry: public.ecr.aws
6461
env:
6562
AWS_REGION: us-east-1
66-
- name: Push Image to Staging ECR
67-
run: |
68-
docker tag opensearch-data-prepper:${{ env.version }} ${{ secrets.ECR_REPOSITORY_URL }}:${{ env.version }}-${{ github.run_number }}
69-
docker push ${{ secrets.ECR_REPOSITORY_URL }}:${{ env.version }}-${{ github.run_number }}
63+
- name: Build and Push Multi-Architecture Docker Image to Staging ECR
64+
run: ./gradlew :release:docker:dockerMultiArchitecture -PdockerRepository=${{ secrets.ECR_REPOSITORY_URL }}:${{ env.version }}-${{ github.run_number }}
7065

7166
validate-docker:
7267
runs-on: ubuntu-latest

‎release/docker/Dockerfile‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,19 @@
1+
FROM public.ecr.aws/amazonlinux/amazonlinux:2023 AS builder
2+
3+
ARG TARGETARCH
4+
5+
RUN dnf -y install tar gzip
6+
7+
# Copy all archives and extract only the correct one
8+
COPY *.tar.gz /tmp/
9+
RUN tar -xzf /tmp/*-linux-${TARGETARCH}.tar.gz -C /tmp && \
10+
rm -f /tmp/*.tar.gz && \
11+
mv /tmp/opensearch-data-prepper-* /tmp/data-prepper
12+
113
FROM public.ecr.aws/amazonlinux/amazonlinux:2023
214

315
ARG PIPELINE_FILEPATH
416
ARG CONFIG_FILEPATH
5-
ARG ARCHIVE_FILE
6-
ARG ARCHIVE_FILE_UNPACKED
717

818
ENV DATA_PREPPER_PATH=/usr/share/data-prepper
919
ENV ENV_CONFIG_FILEPATH=$CONFIG_FILEPATH
@@ -22,8 +32,9 @@ ADD adoptium.repo /etc/yum.repos.d/adoptium.repo
2232
RUN dnf -y install temurin-17-jdk
2333

2434
RUN mkdir -p /var/log/data-prepper
25-
ADD $ARCHIVE_FILE /usr/share
26-
RUN mv /usr/share/$ARCHIVE_FILE_UNPACKED /usr/share/data-prepper
35+
36+
# Copy extracted data-prepper from builder stage
37+
COPY --from=builder /tmp/data-prepper /usr/share/data-prepper
2738

2839
COPY default-data-prepper-config.yaml $ENV_CONFIG_FILEPATH
2940
COPY default-keystore.p12 /usr/share/data-prepper/keystore.p12

‎release/docker/build.gradle‎

Lines changed: 108 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,117 @@
11
/*
22
* Copyright OpenSearch Contributors
33
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
48
*/
59

6-
plugins {
7-
id 'com.palantir.docker' version '0.35.0'
10+
def supportedArchitectures = architectures.get('linux') as String[]
11+
12+
class DockerBuildxTask extends Exec {
13+
@Input
14+
String platform
15+
16+
@Input
17+
String tag
18+
19+
@Input
20+
boolean push = false
21+
22+
@TaskAction
23+
@Override
24+
protected void exec() {
25+
def args = [
26+
'docker', 'buildx', 'build',
27+
'--platform', platform,
28+
'--build-arg', 'CONFIG_FILEPATH=/usr/share/data-prepper/config/data-prepper-config.yaml',
29+
'--build-arg', 'PIPELINE_FILEPATH=/usr/share/data-prepper/pipelines/pipelines.yaml',
30+
'-t', tag
31+
]
32+
33+
if (push) {
34+
args << '--push'
35+
} else if (!platform.contains(',')) {
36+
// Only use --load for single-arch builds
37+
args << '--load'
38+
}
39+
40+
args << '.'
41+
42+
commandLine args
43+
super.exec()
44+
}
45+
}
46+
47+
tasks.register('dockerBuildxSetup', Exec) {
48+
commandLine 'docker', 'buildx', 'create', '--name', 'data-prepper-builder', '--use'
49+
ignoreExitValue = true
50+
}
51+
52+
tasks.register('dockerPrepare') {
53+
dependsOn ':release:releasePrerequisites'
54+
supportedArchitectures.each { arch ->
55+
dependsOn ":release:archives:linux:linux${arch}DistTar"
56+
}
57+
58+
doLast {
59+
supportedArchitectures.each { arch ->
60+
def archiveTask = project(':release:archives:linux').tasks.getByName("linux${arch}DistTar")
61+
def dockerArch = arch == 'x64' ? 'amd64' : arch
62+
def sourceFile = archiveTask.archiveFile.get().asFile
63+
def targetFile = new File("${buildDir}/docker", sourceFile.name.replace("-linux-${arch}", "-linux-${dockerArch}"))
64+
65+
copy {
66+
from sourceFile
67+
into "${buildDir}/docker"
68+
rename { targetFile.name }
69+
}
70+
}
71+
copy {
72+
from "${project.projectDir}/config/default-data-prepper-config.yaml",
73+
"${project.projectDir}/config/default-keystore.p12",
74+
'adoptium.repo',
75+
'Dockerfile'
76+
into "${buildDir}/docker"
77+
}
78+
}
79+
}
80+
81+
tasks.register('docker', DockerBuildxTask) {
82+
dependsOn dockerPrepare
83+
workingDir "${buildDir}/docker"
84+
85+
def currentArch = System.getProperty("os.arch")
86+
platform = currentArch == 'aarch64' ? 'linux/arm64' : 'linux/amd64'
87+
tag = "${project.rootProject.name}:${project.version}"
88+
push = false
89+
}
90+
91+
tasks.register('dockerMultiArchitecture', DockerBuildxTask) {
92+
dependsOn dockerPrepare, dockerBuildxSetup
93+
workingDir "${buildDir}/docker"
94+
95+
platform = supportedArchitectures.collect { arch ->
96+
arch == 'x64' ? 'linux/amd64' : "linux/${arch}"
97+
}.join(',')
98+
tag = "${project.rootProject.name}:${project.version}"
99+
push = false
8100
}
9101

10-
docker {
11-
name "${project.rootProject.name}:${project.version}"
12-
tag "${project.rootProject.name}", "${project.version}"
13-
files project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archiveFile.get().asFile
14-
files "${project.projectDir}/config/default-data-prepper-config.yaml", "${project.projectDir}/config/default-keystore.p12"
15-
files 'adoptium.repo'
16-
buildArgs(['ARCHIVE_FILE' : project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archiveFileName.get(),
17-
'ARCHIVE_FILE_UNPACKED' : project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archiveFileName.get().replace('.tar.gz', ''),
18-
'CONFIG_FILEPATH' : '/usr/share/data-prepper/config/data-prepper-config.yaml',
19-
'PIPELINE_FILEPATH' : '/usr/share/data-prepper/pipelines/pipelines.yaml'])
20-
dockerfile file('Dockerfile')
102+
tasks.register('dockerPush', Exec) {
103+
dependsOn docker
104+
105+
commandLine 'docker', 'push', "${project.rootProject.name}:${project.version}"
21106
}
22107

23-
dockerPrepare.dependsOn ':release:releasePrerequisites'
24-
dockerPrepare.dependsOn ':release:archives:linux:linuxx64DistTar'
25-
dockerPush.dependsOn docker
108+
tasks.register('dockerMultiArchitecturePush', DockerBuildxTask) {
109+
dependsOn dockerPrepare, dockerBuildxSetup
110+
workingDir "${buildDir}/docker"
111+
112+
platform = supportedArchitectures.collect { arch ->
113+
arch == 'x64' ? 'linux/amd64' : "linux/${arch}"
114+
}.join(',')
115+
tag = project.findProperty('dockerRepository') ?: "${project.rootProject.name}:${project.version}"
116+
push = true
117+
}

0 commit comments

Comments
 (0)