Skip to content

Commit 2e67ed9

Browse files
shumvgoloveshumvgolove
authored
ci/scripts: refactor reproducible builds (#1490)
* ci: build cache, remove 20.04, minor refactor * scripts: update build reproducibility script * script: add postgresql reproducibility * script: no cache when building image * scripts: force symlinks * scripts: also download prebuilt postgresql binary * scripts: add missing app var
1 parent 614fa2b commit 2e67ed9

File tree

3 files changed

+117
-35
lines changed

3 files changed

+117
-35
lines changed

‎.github/workflows/build.yml‎

Lines changed: 55 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,7 @@ jobs:
3434
include:
3535
- os: 22.04
3636
ghc: "8.10.7"
37-
platform_name: 20_04-x86-64
38-
- os: 20.04
39-
ghc: "9.6.3"
40-
platform_name: 20_04-x86-64
37+
platform_name: 22_04-8.10.7
4138
- os: 22.04
4239
ghc: "9.6.3"
4340
platform_name: 22_04-x86-64
@@ -48,17 +45,54 @@ jobs:
4845
- name: Clone project
4946
uses: actions/checkout@v3
5047

51-
- name: Prepare image
52-
shell: bash
53-
run: docker build -f Dockerfile.build --build-arg TAG=${{ matrix.os }} --build-arg GHC=${{ matrix.ghc }} -t local .
48+
- name: Set up Docker Buildx
49+
uses: docker/setup-buildx-action@v3
50+
51+
- name: Build and cache Docker image
52+
uses: docker/build-push-action@v6
53+
with:
54+
context: .
55+
load: true
56+
file: Dockerfile.build
57+
tags: build/${{ matrix.platform_name }}:latest
58+
cache-from: |
59+
type=gha
60+
type=gha,scope=master
61+
cache-to: type=gha,mode=max
62+
build-args: |
63+
TAG=${{ matrix.os }}
64+
GHC=${{ matrix.ghc }}
65+
66+
- name: Cache dependencies
67+
uses: actions/cache@v4
68+
with:
69+
path: |
70+
~/.cabal/store
71+
dist-newstyle
72+
key: ${{ matrix.os }}-${{ hashFiles('cabal.project', 'simplexmq.cabal') }}
5473

5574
- name: Start container
5675
shell: bash
57-
run: docker run -t -d --name builder local
76+
run: |
77+
docker run -t -d \
78+
--name builder \
79+
-v ~/.cabal:/root/.cabal \
80+
-v /home/runner/work/_temp:/home/runner/work/_temp \
81+
-v ${{ github.workspace }}:/project \
82+
build/${{ matrix.platform_name }}:latest
5883
5984
- name: Build smp-server (postgresql) and tests
60-
shell: bash
61-
run: docker exec -t builder sh -c 'cabal build --enable-tests -fserver_postgres && mkdir -p /out && for i in smp-server simplexmq-test; do bin=$(find /project/dist-newstyle -name "$i" -type f -executable); chmod +x "$bin"; mv "$bin" /out/; done; strip /out/smp-server'
85+
shell: docker exec -t builder sh {0}
86+
run: |
87+
cabal update
88+
cabal build --enable-tests -fserver_postgres
89+
mkdir -p /out
90+
for i in smp-server simplexmq-test; do
91+
bin=$(find /project/dist-newstyle -name "$i" -type f -executable)
92+
chmod +x "$bin"
93+
mv "$bin" /out/
94+
done
95+
strip /out/smp-server
6296
6397
- name: Copy simplexmq-test from container
6498
shell: bash
@@ -72,15 +106,23 @@ jobs:
72106
docker cp builder:/out/smp-server ./smp-server-postgres-ubuntu-${{ matrix.platform_name }}
73107
74108
- name: Build everything else (standard)
75-
shell: bash
76-
run: docker exec -t -e apps="$apps" builder sh -c 'cabal build && mkdir -p /out && for i in $apps; do bin=$(find /project/dist-newstyle -name "$i" -type f -executable); strip "$bin"; chmod +x "$bin"; mv "$bin" /out/; done'
109+
shell: docker exec -t builder sh {0}
110+
run: |
111+
cabal build
112+
mkdir -p /out
113+
for i in ${{ env.apps }}; do
114+
bin=$(find /project/dist-newstyle -name "$i" -type f -executable)
115+
strip "$bin"
116+
chmod +x "$bin"
117+
mv "$bin" /out/
118+
done
77119
78120
- name: Copy binaries from container and prepare them
79121
if: startsWith(github.ref, 'refs/tags/v')
80122
shell: bash
81123
run: |
82124
docker cp builder:/out .
83-
for i in $apps; do mv ./out/$i ./$i-ubuntu-${{ matrix.platform_name }}; done
125+
for i in ${{ env.apps }}; do mv ./out/$i ./$i-ubuntu-${{ matrix.platform_name }}; done
84126
85127
- name: Build changelog
86128
if: startsWith(github.ref, 'refs/tags/v')

‎Dockerfile.build‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,4 @@ ENV PATH="/root/.cabal/bin:/root/.ghcup/bin:$PATH"
2828
RUN ghcup set ghc "${GHC}" && \
2929
ghcup set cabal "${CABAL}"
3030

31-
# Copy only the source code
32-
COPY apps /project/apps/
33-
COPY cbits /project/cbits/
34-
COPY src /project/src/
35-
COPY tests /project/tests/
36-
37-
COPY cabal.project Setup.hs simplexmq.cabal LICENSE /project
38-
3931
WORKDIR /project
40-
41-
# Compile app
42-
RUN cabal update

‎scripts/reproduce-builds.sh‎

Lines changed: 62 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,81 @@
11
#!/usr/bin/env sh
22
set -eu
33

4-
tag="$1"
4+
TAG="$1"
55

6-
git clone https://github.com/simplex-chat/simplexmq && cd simplexmq
6+
tempdir="$(mktemp -d)"
7+
init_dir="$PWD"
78

8-
git checkout "$tag"
9+
mkdir -p "$init_dir/$TAG/from-source" "$init_dir/$TAG/prebuilt"
10+
11+
git -C "$tempdir" clone https://github.com/simplex-chat/simplexmq.git &&\
12+
cd "$tempdir/simplexmq" &&\
13+
git checkout "$TAG"
914

1015
for os in 20.04 22.04 24.04; do
11-
mkdir -p out-${os}-github;
16+
os_url="$(printf '%s' "$os" | tr '.' '_')"
17+
mkdir -p "$init_dir/cache/cabal/builder-${os}" "$init_dir/cache/dist-newstyle/builder-${os}"
18+
chmod g+wX "$init_dir/cache"
19+
20+
docker build \
21+
--no-cache \
22+
-f "$tempdir/simplexmq/Dockerfile.build" \
23+
--build-arg TAG=${os} \
24+
-t repro-${os} \
25+
.
1226

13-
docker build -f Dockerfile.build --build-arg TAG=${os} -t repro-${os} .
14-
docker run -t -d --name builder-${os} repro-${os}
27+
docker run \
28+
-t \
29+
-d \
30+
-v "$init_dir/cache/cabal/builder-${os}:/root/.cabal" \
31+
-v "$init_dir/cache/dist-newstyle/builder-${os}:/dist-newstyle" \
32+
-v "$tempdir/simplexmq:/project" \
33+
--name builder-${os} \
34+
repro-${os}
1535

16-
apps='smp-server xftp-server ntf-server xftp'
17-
os_url="$(printf '%s' "$os" | tr '.' '_')"
1836

19-
docker exec -t -e apps="$apps" builder-${os} sh -c 'cabal build && mkdir /out && for i in $apps; do bin=$(find /project/dist-newstyle -name "$i" -type f -executable); strip "$bin"; chmod +x "$bin"; mv "$bin" /out/; done'
37+
apps='smp-server xftp-server ntf-server xftp'
2038

21-
docker cp builder-${os}:/out out-${os}
39+
# Regular build (all)
40+
docker exec \
41+
-t \
42+
-e apps="$apps" \
43+
builder-${os} \
44+
sh -c 'ln -fs /dist-newstyle ./dist-newstyle && cabal update && cabal build && mkdir -p /out && for i in $apps; do bin=$(find /dist-newstyle -name "$i" -type f -executable); strip "$bin"; chmod +x "$bin"; mv "$bin" /out/; done'
45+
46+
docker cp \
47+
builder-${os}:/out \
48+
out-${os}
49+
50+
# PostgreSQL build (only smp-server)
51+
docker exec \
52+
-t \
53+
builder-${os} \
54+
sh -c 'ln -fs /dist-newstyle ./dist-newstyle && cabal update && cabal build -fserver_postgres exe:smp-server && mkdir -p /out && bin=$(find /dist-newstyle -name "smp-server" -type f -executable); strip "$bin"; chmod +x "$bin"; mv "$bin" /out/'
55+
56+
docker cp \
57+
builder-${os}:/out/smp-server \
58+
"$init_dir/$TAG/from-source/smp-server-postgres-ubuntu-${os_url}-x86-64"
59+
60+
curl -L \
61+
--output-dir "$init_dir/$TAG/prebuilt/" \
62+
-O \
63+
"https://github.com/simplex-chat/simplexmq/releases/download/${TAG}/smp-server-postgres-ubuntu-${os_url}-x86-64"
2264

2365
for app in $apps; do
24-
curl -L "https://github.com/simplex-chat/simplexmq/releases/download/${tag}/${app}-ubuntu-${os_url}-x86-64" -o out-${os}-github/${app}
66+
curl -L \
67+
--output-dir "$init_dir/$TAG/prebuilt/" \
68+
-O \
69+
"https://github.com/simplex-chat/simplexmq/releases/download/${TAG}/${app}-ubuntu-${os_url}-x86-64"
70+
71+
mv "./out-${os}/$app" "$init_dir/$TAG/from-source/${app}-ubuntu-${os_url}-x86-64"
2572
done
2673

2774
docker stop builder-${os}
2875
docker rm builder-${os}
2976
docker image rm repro-${os}
3077
done
78+
79+
# Cleanup
80+
cd "$init_dir"
81+
rm -rf "$tempdir"

0 commit comments

Comments
 (0)