Skip to content

Commit be47232

Browse files
sebthomsebthom
committed
ci: improve multi-arch builds
1 parent 2b94703 commit be47232

File tree

8 files changed

+281
-214
lines changed

8 files changed

+281
-214
lines changed

‎.editorconfig‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ end_of_line = lf
99
insert_final_newline = true
1010
trim_trailing_whitespace = true
1111
indent_style = space
12-
indent_size = 3
12+
indent_size = 2
1313

1414
[*.{bat,cmd}]
1515
end_of_line = crlf

‎.github/workflows/build.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ jobs:
5353
strategy:
5454
matrix:
5555
SOFTHSM_VERSION: [ "latest", "develop" ]
56-
DOCKER_BASE_IMAGE: [ alpine:latest, "debian:stable-slim" ]
56+
DOCKER_BASE_IMAGE: [ alpine:3, "debian:stable-slim" ]
5757

5858
steps:
5959
- name: "Show: GitHub context"

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,6 @@ nb-configuration.xml
3535
# patch
3636
*.orig
3737
*.rej
38+
39+
# nektos/act
40+
.actrc

‎CODE_OF_CONDUCT.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ representative at an online or offline event.
6060

6161
Instances of abusive, harassing, or otherwise unacceptable behavior may be
6262
reported to the community leaders responsible for enforcement at
63-
https://vegardit.com/about/legal/.
63+
https://vegardit.com/en/legal/.
6464
All complaints will be reviewed and investigated promptly and fairly.
6565

6666
All community leaders are obligated to respect the privacy and security of the

‎README.md‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,11 @@
1616

1717
> **SoftHSM has been developed for development purposes only. Don't use in production!**
1818
19-
Multi-arch Docker image to run a virtual HSM (Hardware Security Module) network service based on [SoftHSM2](https://github.com/softhsm/SoftHSMv2) and
19+
A multi-arch Docker image to run a virtual HSM (Hardware Security Module) network service based on [SoftHSM2](https://github.com/softhsm/SoftHSMv2) and
2020
[pkcs11-proxy](https://github.com/SUNET/pkcs11-proxy/).
2121

22+
Automatically rebuilt **weekly** to include the latest OS security fixes.
23+
2224
Client applications can communicate with the HSM via TCP/TLS using libpkcs11-proxy.so and an OpenSSL TLS-PSK:
2325

2426
![](doc/aod.png)
@@ -28,11 +30,11 @@ Client applications can communicate with the HSM via TCP/TLS using libpkcs11-pro
2830

2931
|Tag|Description|Base Image
3032
|-|-|-
31-
|`:latest` <br> `:latest-alpine` | weekly build of the latest available SoftHSM release | alpine:latest
33+
|`:latest` <br> `:latest-alpine` | weekly build of the latest available SoftHSM release | alpine:3
3234
|`:latest-debian` | weekly build of the latest available SoftHSM release | debian:stable-slim
33-
|`:develop` <br> `:develop-alpine` | weekly build of the development branch | alpine:latest
35+
|`:develop` <br> `:develop-alpine` | weekly build of the development branch | alpine:3
3436
|`:develop-debian` | weekly build of the development branch | debian:stable-slim
35-
|`:2.x` <br> `:2.x-alpine` | weekly build of the latest minor version of the respective <br> major release, e.g. `2.x` may contain release `2.6` | alpine:latest
37+
|`:2.x` <br> `:2.x-alpine` | weekly build of the latest minor version of the respective <br> major release, e.g. `2.x` may contain release `2.6` | alpine:3
3638
|`:2.x-debian` | weekly build of the latest minor version of the respective <br> major release, e.g. `2.x` may contain release `2.6` | debian:stable-slim
3739

3840
See all tags at https://hub.docker.com/r/vegardit/softhsm2-pkcs11-proxy/tags

‎build-image.sh‎

Lines changed: 156 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -5,174 +5,218 @@
55
# SPDX-License-Identifier: Apache-2.0
66
# SPDX-ArtifactOfProjectHomePage: https://github.com/vegardit/docker-softhsm2-pkcs11-proxy
77

8-
function curl() {
9-
command curl -sSfL --connect-timeout 10 --max-time 30 --retry 3 --retry-all-errors "$@"
10-
}
11-
128
shared_lib="$(dirname "${BASH_SOURCE[0]}")/.shared"
13-
[[ -e $shared_lib ]] || curl "https://raw.githubusercontent.com/vegardit/docker-shared/v1/download.sh?_=$(date +%s)" | bash -s v1 "$shared_lib" || exit 1
9+
[[ -e $shared_lib ]] || curl -sSfL "https://raw.githubusercontent.com/vegardit/docker-shared/v1/download.sh?_=$(date +%s)" | bash -s v1 "$shared_lib" || exit 1
1410
# shellcheck disable=SC1091 # Not following: $shared_lib/lib/build-image-init.sh was not specified as input
1511
source "$shared_lib/lib/build-image-init.sh"
1612

1713

1814
#################################################
19-
# specify target image repo/tag
15+
# declare image meta
2016
#################################################
2117
image_repo=${DOCKER_IMAGE_REPO:-vegardit/softhsm2-pkcs11-proxy}
22-
base_image_name=${DOCKER_BASE_IMAGE:-alpine:3}
23-
case $base_image_name in
24-
*alpine*) base_image_linux_flavor=alpine ;;
25-
*debian*) base_image_linux_flavor=debian ;;
26-
*) echo "ERROR: Unsupported base image $base_image_name"; exit 1 ;;
18+
base_image=${DOCKER_BASE_IMAGE:-alpine:3}
19+
case $base_image in
20+
*alpine*) base_image_linux_flavor=alpine ;;
21+
*debian*) base_image_linux_flavor=debian ;;
22+
*) echo "ERROR: Unsupported base image $base_image"; exit 1 ;;
2723
esac
2824

2925
app_version=${SOFTHSM_VERSION:-latest}
3026
case $app_version in \
31-
latest)
32-
#app_version=$(curl https://github.com/softhsm/SoftHSMv2/releases/latest | sed -n "s/.*releases\/tag\/\([0-9]\.[0-9]\.[0-9]\)['\"].*/\1/p" | head -1)
33-
app_version=$(curl https://github.com/softhsm/SoftHSMv2/tags | sed -n "s/.*releases\/tag\/\([0-9]\.[0-9]\.[0-9]\)['\"].*/\1/p" | head -1)
34-
softhsm_source_url=https://codeload.github.com/softhsm/SoftHSMv2/tar.gz/refs/tags/$app_version
35-
app_version_is_latest=1
36-
;;
37-
develop)
38-
softhsm_source_url=https://codeload.github.com/softhsm/SoftHSMv2/tar.gz/refs/heads/develop
39-
;;
40-
*)
41-
softhsm_source_url=https://codeload.github.com/softhsm/SoftHSMv2/tar.gz/refs/tags/$app_version
42-
;;
27+
latest)
28+
#app_version=$(curl https://github.com/softhsm/SoftHSMv2/releases/latest | sed -n "s/.*releases\/tag\/\([0-9]\.[0-9]\.[0-9]\)['\"].*/\1/p" | head -1)
29+
app_version=$(curl https://github.com/softhsm/SoftHSMv2/tags | sed -n "s/.*releases\/tag\/\([0-9]\.[0-9]\.[0-9]\)['\"].*/\1/p" | head -1)
30+
softhsm_source_url=https://codeload.github.com/softhsm/SoftHSMv2/tar.gz/refs/tags/$app_version
31+
app_version_is_latest=1
32+
;;
33+
develop) softhsm_source_url=https://codeload.github.com/softhsm/SoftHSMv2/tar.gz/refs/heads/develop ;;
34+
*) softhsm_source_url=https://codeload.github.com/softhsm/SoftHSMv2/tar.gz/refs/tags/$app_version ;;
4335
esac
4436
log INFO "app_version=$app_version"
4537
log INFO "softhsm_source_url=$softhsm_source_url"
4638

39+
platforms="linux/amd64,linux/arm64/v8" # linux/arm/v7
4740

48-
#################################################
49-
# calculate tags
50-
#################################################
51-
declare -a tags=()
41+
declare -A image_meta=(
42+
[authors]="Vegard IT GmbH (vegardit.com)"
43+
[title]="$image_repo"
44+
[description]="Docker image to run a virtual HSM (Hardware Security Module) network service based on SoftHSM2 and pkcs11-proxy"
45+
[source]="$(git config --get remote.origin.url)"
46+
[revision]="$(git rev-parse --short HEAD)"
47+
[version]="$(git rev-parse --short HEAD)"
48+
[created]="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
49+
)
5250

51+
declare -a tags=()
5352
if [[ $app_version == develop ]]; then
54-
tags+=("$image_repo:develop-$base_image_linux_flavor") # :develop-alpine
55-
if [[ $base_image_linux_flavor == alpine ]]; then
56-
tags+=("$image_repo:develop") # :develop
57-
fi
53+
tags+=("develop-$base_image_linux_flavor") # :develop-alpine
54+
if [[ $base_image_linux_flavor == alpine ]]; then
55+
tags+=("develop") # :develop
56+
fi
5857
else
59-
if [[ $app_version =~ ^[0-9]+\..*$ ]]; then
60-
tags+=("$image_repo:${app_version%%.*}.x-$base_image_linux_flavor") # :2.x-alpine
61-
if [[ $base_image_linux_flavor == alpine ]]; then
62-
tags+=("$image_repo:${app_version%%.*}.x-$base_image_linux_flavor") # :2.x
63-
fi
64-
fi
65-
66-
if [[ ${app_version_is_latest:-} == 1 ]]; then
67-
tags+=("$image_repo:latest-$base_image_linux_flavor") # :latest-alpine
68-
if [[ $base_image_linux_flavor == alpine ]]; then
69-
tags+=("$image_repo:latest") # :latest
70-
fi
71-
fi
58+
if [[ $app_version =~ ^[0-9]+\..*$ ]]; then
59+
tags+=("${app_version%%.*}.x-$base_image_linux_flavor") # :2.x-alpine
60+
if [[ $base_image_linux_flavor == alpine ]]; then
61+
tags+=("${app_version%%.*}.x-$base_image_linux_flavor") # :2.x
62+
fi
63+
fi
64+
if [[ ${app_version_is_latest:-} == 1 ]]; then
65+
tags+=("latest-$base_image_linux_flavor") # :latest-alpine
66+
if [[ $base_image_linux_flavor == alpine ]]; then
67+
tags+=("latest") # :latest
68+
fi
69+
fi
7270
fi
7371

74-
tag_args=()
75-
for t in "${tags[@]}"; do
76-
tag_args+=( --tag "$t" )
77-
done
78-
79-
image_name=${tags[0]}
80-
8172

8273
#################################################
83-
# build the image
74+
# decide if multi-arch build
8475
#################################################
85-
log INFO "Building docker image [$image_name]..."
86-
if [[ $OSTYPE == "cygwin" || $OSTYPE == "msys" ]]; then
87-
project_root=$(cygpath -w "$project_root")
76+
if [[ ${DOCKER_PUSH:-} == "true" || ${DOCKER_PUSH_GHCR:-} == "true" ]]; then
77+
build_multi_arch="true"
8878
fi
8979

90-
case $base_image_name in
91-
*alpine*) dockerfile="alpine.Dockerfile" ;;
92-
*debian*) dockerfile="debian.Dockerfile" ;;
93-
*) echo "ERROR: Unsupported base image $base_image_name"; exit 1 ;;
94-
esac
80+
81+
#################################################
82+
# prepare docker
83+
#################################################
84+
run_step -- docker version
9585

9686
# https://github.com/docker/buildx/#building-multi-platform-images
97-
set -x
87+
run_step -- docker buildx version # ensures buildx is enabled
9888

99-
docker --version
10089
export DOCKER_BUILDKIT=1
101-
export DOCKER_CLI_EXPERIMENTAL=1 # prevents "docker: 'buildx' is not a docker command."
90+
export DOCKER_CLI_EXPERIMENTAL=1 # prevents "docker: 'buildx' is not a docker command." in older Docker versions
91+
92+
if [[ ${build_multi_arch:-} == "true" ]]; then
93+
# Use a temporary local registry to work around Docker/Buildx/BuildKit quirks,
94+
# enabling us to build/test multiarch images locally before pushing.
95+
run_step -- start_docker_registry LOCAL_REGISTRY
10296

103-
# Register QEMU emulators for all architectures so Docker can run and build multi-arch images
104-
docker run --privileged --rm ghcr.io/dockerhub-mirror/tonistiigi__binfmt --install all
97+
# Register QEMU emulators so Docker can run and build multi-arch images
98+
run_step "Install QEMU emulators" -- \
99+
docker run --privileged --rm ghcr.io/dockerhub-mirror/tonistiigi__binfmt --install all
100+
fi
105101

106102
# https://docs.docker.com/build/buildkit/configure/#resource-limiting
107103
echo "
108104
[worker.oci]
109105
max-parallelism = 3
110106
" | sudo tee /etc/buildkitd.toml
111107

112-
docker buildx version # ensures buildx is enabled
113-
docker buildx create --config /etc/buildkitd.toml --use # prevents: error: multiple platforms feature is currently not supported for docker driver. Please switch to a different driver (eg. "docker buildx create --use")
114-
trap 'docker buildx stop' EXIT
115-
# shellcheck disable=SC2154,SC2046 # base_layer_cache_key is referenced but not assigned / Quote this to prevent word splitting
116-
docker buildx build "$project_root" \
117-
--file "image/$dockerfile" \
118-
--progress=plain \
119-
--pull \
120-
--build-arg INSTALL_SUPPORT_TOOLS="${INSTALL_SUPPORT_TOOLS:-0}" \
121-
`# using the current date as value for BASE_LAYER_CACHE_KEY, i.e. the base layer cache (that holds system packages with security updates) will be invalidate once per day` \
122-
--build-arg BASE_LAYER_CACHE_KEY="$base_layer_cache_key" \
123-
--build-arg BASE_IMAGE="$base_image_name" \
124-
--build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
125-
--build-arg GIT_BRANCH="${GIT_BRANCH:-$(git rev-parse --abbrev-ref HEAD)}" \
126-
--build-arg GIT_COMMIT_DATE="$(date -d "@$(git log -1 --format='%at')" --utc +'%Y-%m-%d %H:%M:%S UTC')" \
127-
--build-arg GIT_COMMIT_HASH="$(git rev-parse --short HEAD)" \
128-
--build-arg GIT_REPO_URL="$(git config --get remote.origin.url)" \
129-
--build-arg SOFTHSM_SOURCE_URL="$softhsm_source_url" \
130-
--build-arg PKCS11_PROXY_SOURCE_URL="https://codeload.github.com/smallstep/pkcs11-proxy/tar.gz/refs/heads/master" \
131-
`#--build-arg PKCS11_PROXY_SOURCE_URL="https://codeload.github.com/scobiej/pkcs11-proxy/tar.gz/refs/heads/osx-openssl1-1"` \
132-
`#--build-arg PKCS11_PROXY_SOURCE_URL="https://codeload.github.com/SUNET/pkcs11-proxy/tar.gz/refs/heads/master"` \
133-
$(if [[ ${ACT:-} == "true" || ${DOCKER_PUSH:-} != "true" ]]; then \
134-
echo -n "--load --output type=docker"; \
135-
else \
136-
echo -n "--platform linux/amd64,linux/arm64" `# ,linux/arm/v7"`; \
137-
fi) \
138-
"${tag_args[@]}" \
139-
$(if [[ ${DOCKER_PUSH:-} == "true" ]]; then echo -n "--push"; fi) \
140-
"$@"
141-
set +x
108+
builder_name="bx-$(date +%s)-$RANDOM"
109+
run_step "Configure buildx builder" -- docker buildx create \
110+
--name "$builder_name" \
111+
--bootstrap \
112+
--config /etc/buildkitd.toml \
113+
--driver-opt network=host `# required for buildx to access the temporary registry` \
114+
--driver docker-container \
115+
--driver-opt image=ghcr.io/dockerhub-mirror/moby__buildkit:latest
116+
trap 'docker buildx rm --force "$builder_name"' EXIT
142117

143-
if [[ ${DOCKER_PUSH:-} == "true" ]]; then
144-
docker image pull "$image_name"
118+
119+
#################################################
120+
# build the image
121+
#################################################
122+
image_name=image_repo:${tags[0]}
123+
124+
case $base_image in
125+
*alpine*) dockerfile="alpine.Dockerfile" ;;
126+
*debian*) dockerfile="debian.Dockerfile" ;;
127+
*) echo "ERROR: Unsupported base image $base_image"; exit 1 ;;
128+
esac
129+
130+
build_opts=(
131+
--file "image/$dockerfile"
132+
--builder "$builder_name"
133+
--progress=plain
134+
--pull
135+
# using the current date as value for BASE_LAYER_CACHE_KEY, i.e. the base layer cache (that holds system packages with security updates) will be invalidate once per day
136+
--build-arg BASE_LAYER_CACHE_KEY="$base_layer_cache_key"
137+
--build-arg BASE_IMAGE="$base_image"
138+
--build-arg GIT_BRANCH="${GIT_BRANCH:-$(git rev-parse --abbrev-ref HEAD)}"
139+
--build-arg GIT_COMMIT_DATE="$(date -d "@$(git log -1 --format='%at')" --utc +'%Y-%m-%d %H:%M:%S UTC')"
140+
--build-arg SOFTHSM_SOURCE_URL="$softhsm_source_url"
141+
--build-arg PKCS11_PROXY_SOURCE_URL="https://codeload.github.com/smallstep/pkcs11-proxy/tar.gz/refs/heads/master"
142+
#--build-arg PKCS11_PROXY_SOURCE_URL="https://codeload.github.com/scobiej/pkcs11-proxy/tar.gz/refs/heads/osx-openssl1-1"
143+
#--build-arg PKCS11_PROXY_SOURCE_URL="https://codeload.github.com/SUNET/pkcs11-proxy/tar.gz/refs/heads/master"
144+
--build-arg INSTALL_SUPPORT_TOOLS="${INSTALL_SUPPORT_TOOLS:-0}"
145+
)
146+
147+
for key in "${!image_meta[@]}"; do
148+
build_opts+=(--build-arg "OCI_${key}=${image_meta[$key]}")
149+
if [[ ${build_multi_arch:-} == "true" ]]; then
150+
build_opts+=(--annotation "index:org.opencontainers.image.${key}=${image_meta[$key]}")
151+
fi
152+
done
153+
154+
if [[ ${build_multi_arch:-} == "true" ]]; then
155+
build_opts+=(--push)
156+
build_opts+=(--sbom=true) # https://docs.docker.com/build/metadata/attestations/sbom/#create-sbom-attestations
157+
build_opts+=(--platform "$platforms")
158+
build_opts+=(--tag "$LOCAL_REGISTRY/$image_name")
159+
else
160+
build_opts+=(--output "type=docker,load=true")
161+
build_opts+=(--tag "$image_name")
145162
fi
146163

164+
if [[ $OSTYPE == "cygwin" || $OSTYPE == "msys" ]]; then
165+
project_root=$(cygpath -w "$project_root")
166+
fi
167+
168+
run_step "Building docker image [$image_name]..." -- \
169+
docker buildx build "${build_opts[@]}" "$project_root"
170+
147171

148172
#################################################
149-
# test image
173+
# load image into local docker daemon for testing
150174
#################################################
151-
echo
152-
log INFO "Testing docker image [$image_name]..."
153-
(set -x; docker run --rm "$image_name" /usr/local/bin/softhsm2-util --version)
154-
echo
175+
if [[ ${build_multi_arch:-} == "true" ]]; then
176+
run_step "Load image into local daemon for testing" @@ "
177+
docker pull '$LOCAL_REGISTRY/$image_name';
178+
docker tag '$LOCAL_REGISTRY/$image_name' '$image_name'
179+
"
180+
fi
155181

156182

157183
#################################################
158184
# perform security audit
159185
#################################################
160186
if [[ ${DOCKER_AUDIT_IMAGE:-1} == "1" ]]; then
187+
run_step "Auditing docker image [$image_name]" -- \
161188
bash "$shared_lib/cmd/audit-image.sh" "$image_name"
162189
fi
163190

164191

165192
#################################################
166-
# push image to ghcr.io
193+
# test image
194+
#################################################
195+
run_step "Testing docker image [$image_name]" -- \
196+
docker run --pull=never --rm "$image_name" /usr/local/bin/softhsm2-util --version
197+
198+
199+
#################################################
200+
# push image
167201
#################################################
202+
function regctl() {
203+
run_step "regctl ${*}" -- \
204+
docker run --rm \
205+
-u "$(id -u):$(id -g)" -e HOME -v "$HOME:$HOME" \
206+
-v /etc/docker/certs.d:/etc/docker/certs.d:ro \
207+
--network host `# required to access the temporary registry` \
208+
ghcr.io/regclient/regctl:latest \
209+
--host "reg=$LOCAL_REGISTRY,tls=disabled" \
210+
"${@}"
211+
}
212+
213+
if [[ ${DOCKER_PUSH:-} == "true" ]]; then
214+
for tag in "${tags[@]}"; do
215+
regctl image copy --referrers "$LOCAL_REGISTRY/$image_name" "docker.io/$image_repo:$tag"
216+
done
217+
fi
168218
if [[ ${DOCKER_PUSH_GHCR:-} == "true" ]]; then
169-
for tag in "${tags[@]}"; do
170-
set -x
171-
docker run --rm \
172-
-u "$(id -u):$(id -g)" -e HOME -v "$HOME:$HOME" \
173-
-v /etc/docker/certs.d:/etc/docker/certs.d:ro \
174-
ghcr.io/regclient/regctl:latest \
175-
image copy "$tag" "ghcr.io/$tag"
176-
set +x
177-
done
219+
for tag in "${tags[@]}"; do
220+
regctl image copy --referrers "$LOCAL_REGISTRY/$image_name" "ghcr.io/$image_repo:$tag"
221+
done
178222
fi

0 commit comments

Comments
 (0)