Skip to content

Commit 65b83c4

Browse files
xmatthiasxmatthias
authored
Merge pull request freqtrade#12208 from freqtrade/test_flat_dockerbuild
Restructure docker builds
2 parents a7e7c17 + 31204a3 commit 65b83c4

File tree

4 files changed

+297
-237
lines changed

4 files changed

+297
-237
lines changed

‎.github/actions/docker-tags/action.yml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
name: 'docker-tags'
2+
description: 'Set Docker default Tag environment variables'
3+
# inputs:
4+
outputs:
5+
BRANCH_NAME:
6+
description: 'The branch name'
7+
value: ${{ steps.tags.outputs.BRANCH_NAME }}
8+
TAG:
9+
description: 'The Docker tag'
10+
value: ${{ steps.tags.outputs.TAG }}
11+
TAG_PLOT:
12+
description: 'The Docker tag for the plot'
13+
value: ${{ steps.tags.outputs.TAG_PLOT }}
14+
TAG_FREQAI:
15+
description: 'The Docker tag for the freqai'
16+
value: ${{ steps.tags.outputs.TAG_FREQAI }}
17+
TAG_FREQAI_RL:
18+
description: 'The Docker tag for the freqai_rl'
19+
value: ${{ steps.tags.outputs.TAG_FREQAI_RL }}
20+
TAG_FREQAI_TORCH:
21+
description: 'The Docker tag for the freqai_torch'
22+
value: ${{ steps.tags.outputs.TAG_FREQAI_TORCH }}
23+
TAG_ARM:
24+
description: 'The Docker tag for the arm'
25+
value: ${{ steps.tags.outputs.TAG_ARM }}
26+
TAG_PLOT_ARM:
27+
description: 'The Docker tag for the plot arm'
28+
value: ${{ steps.tags.outputs.TAG_PLOT_ARM }}
29+
TAG_FREQAI_ARM:
30+
description: 'The Docker tag for the freqai arm'
31+
value: ${{ steps.tags.outputs.TAG_FREQAI_ARM }}
32+
TAG_FREQAI_RL_ARM:
33+
description: 'The Docker tag for the freqai_rl arm'
34+
value: ${{ steps.tags.outputs.TAG_FREQAI_RL_ARM }}
35+
TAG_PI:
36+
description: 'The Docker tag for the pi'
37+
value: ${{ steps.tags.outputs.TAG_PI }}
38+
CACHE_TAG_PI:
39+
description: 'The Docker cache tag for the pi'
40+
value: ${{ steps.tags.outputs.CACHE_TAG_PI }}
41+
runs:
42+
using: "composite"
43+
steps:
44+
- name: Extract branch name
45+
shell: bash
46+
id: tags
47+
env:
48+
BRANCH_NAME_INPUT: ${{ github.event.inputs.branch_name }}
49+
run: |
50+
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
51+
BRANCH_NAME="${BRANCH_NAME_INPUT}"
52+
else
53+
BRANCH_NAME="${GITHUB_REF##*/}"
54+
fi
55+
56+
# Replace / with _ to create a valid tag
57+
TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
58+
TAG_PLOT=${TAG}_plot
59+
TAG_FREQAI=${TAG}_freqai
60+
TAG_FREQAI_RL=${TAG_FREQAI}rl
61+
TAG_FREQAI_TORCH=${TAG_FREQAI}torch
62+
63+
TAG_ARM=${TAG}_arm
64+
TAG_PLOT_ARM=${TAG_PLOT}_arm
65+
TAG_FREQAI_ARM=${TAG_FREQAI}_arm
66+
TAG_FREQAI_RL_ARM=${TAG_FREQAI_RL}_arm
67+
68+
TAG_PI="${TAG}_pi"
69+
70+
CACHE_TAG_PI=${CACHE_IMAGE}:${TAG_PI}_cache
71+
72+
echo "BRANCH_NAME=${BRANCH_NAME}" >> "$GITHUB_OUTPUT"
73+
echo "TAG=${TAG}" >> "$GITHUB_OUTPUT"
74+
echo "TAG_PLOT=${TAG_PLOT}" >> "$GITHUB_OUTPUT"
75+
echo "TAG_FREQAI=${TAG_FREQAI}" >> "$GITHUB_OUTPUT"
76+
echo "TAG_FREQAI_RL=${TAG_FREQAI_RL}" >> "$GITHUB_OUTPUT"
77+
echo "TAG_FREQAI_TORCH=${TAG_FREQAI_TORCH}" >> "$GITHUB_OUTPUT"
78+
echo "TAG_ARM=${TAG_ARM}" >> "$GITHUB_OUTPUT"
79+
echo "TAG_PLOT_ARM=${TAG_PLOT_ARM}" >> "$GITHUB_OUTPUT"
80+
echo "TAG_FREQAI_ARM=${TAG_FREQAI_ARM}" >> "$GITHUB_OUTPUT"
81+
echo "TAG_FREQAI_RL_ARM=${TAG_FREQAI_RL_ARM}" >> "$GITHUB_OUTPUT"
82+
echo "TAG_PI=${TAG_PI}" >> "$GITHUB_OUTPUT"
83+
84+
echo "CACHE_TAG_PI=${CACHE_TAG_PI}" >> "$GITHUB_OUTPUT"
85+
86+
cat "$GITHUB_OUTPUT"
87+
88+
- name: Save commit SHA to file
89+
shell: bash
90+
# Add commit to docker container
91+
run: |
92+
echo "${GITHUB_SHA}" > freqtrade_commit

‎.github/workflows/docker-build.yml‎

Lines changed: 205 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ env:
2424
IMAGE_NAME: "freqtradeorg/freqtrade"
2525
CACHE_IMAGE: "freqtradeorg/freqtrade_cache"
2626
GHCR_IMAGE_NAME: "ghcr.io/freqtrade/freqtrade"
27+
PI_PLATFORM: "linux/arm/v7"
2728

2829
jobs:
2930
deploy-docker:
@@ -36,33 +37,20 @@ jobs:
3637
with:
3738
persist-credentials: false
3839

39-
- name: Set up Python
40-
uses: actions/setup-python@v5
41-
with:
42-
python-version: "3.12"
43-
44-
- name: Extract branch name
45-
env:
46-
BRANCH_NAME_INPUT: ${{ github.event.inputs.branch_name }}
47-
run: |
48-
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
49-
BRANCH_NAME="${BRANCH_NAME_INPUT}"
50-
else
51-
BRANCH_NAME="${GITHUB_REF##*/}"
52-
fi
53-
echo "GITHUB_REF='${GITHUB_REF}'"
54-
echo "BRANCH_NAME='${BRANCH_NAME}'"
55-
echo "BRANCH_NAME=${BRANCH_NAME}" >> "$GITHUB_ENV"
40+
- name: Set docker tag names
41+
id: tags
42+
uses: ./.github/actions/docker-tags
5643

5744
- name: Login to Docker Hub
5845
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
5946
with:
6047
username: ${{ secrets.DOCKER_USERNAME }}
6148
password: ${{ secrets.DOCKER_PASSWORD }}
6249

63-
6450
- name: Set up QEMU
6551
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
52+
with:
53+
cache-image: false
6654

6755
- name: Set up Docker Buildx
6856
id: buildx
@@ -73,9 +61,86 @@ jobs:
7361
env:
7462
PLATFORMS: ${{ steps.buildx.outputs.platforms }}
7563

76-
- name: Build and test and push docker images
64+
- name: Build image without cache
65+
if: github.event_name == 'schedule'
66+
env:
67+
TAG: ${{ steps.tags.outputs.TAG }}
68+
run: |
69+
docker build -t ${CACHE_IMAGE}:${TAG} .
70+
71+
- name: Build ARMHF image without cache
72+
if: github.event_name == 'schedule'
73+
env:
74+
TAG_PI: ${{ steps.tags.outputs.TAG_PI }}
75+
CACHE_TAG_PI: ${{ steps.tags.outputs.CACHE_TAG_PI }}
76+
run: |
77+
docker buildx build \
78+
--cache-to=type=registry,ref=${CACHE_TAG_PI} \
79+
-f docker/Dockerfile.armhf \
80+
--platform ${PI_PLATFORM} \
81+
-t ${IMAGE_NAME}:${TAG_PI} \
82+
--push \
83+
--provenance=false \
84+
.
85+
86+
- name: Build image with cache
87+
if: github.event_name != 'schedule'
88+
env:
89+
TAG: ${{ steps.tags.outputs.TAG }}
90+
run: |
91+
docker pull ${IMAGE_NAME}:${TAG} || true
92+
docker build --cache-from ${IMAGE_NAME}:${TAG} -t ${CACHE_IMAGE}:${TAG} .
93+
94+
- name: Build ARMHF image with cache
95+
if: github.event_name != 'schedule'
96+
# disable provenance due to https://github.com/docker/buildx/issues/1509
97+
env:
98+
TAG_PI: ${{ steps.tags.outputs.TAG_PI }}
99+
CACHE_TAG_PI: ${{ steps.tags.outputs.CACHE_TAG_PI }}
100+
run: |
101+
docker buildx build \
102+
--cache-from=type=registry,ref=${CACHE_TAG_PI} \
103+
--cache-to=type=registry,ref=${CACHE_TAG_PI} \
104+
-f docker/Dockerfile.armhf \
105+
--platform ${PI_PLATFORM} \
106+
-t ${IMAGE_NAME}:${TAG_PI} \
107+
--push \
108+
--provenance=false \
109+
.
110+
111+
- name: Run build for AI images
112+
env:
113+
TAG: ${{ steps.tags.outputs.TAG }}
114+
TAG_PLOT: ${{ steps.tags.outputs.TAG_PLOT }}
115+
TAG_FREQAI: ${{ steps.tags.outputs.TAG_FREQAI }}
116+
TAG_FREQAI_RL: ${{ steps.tags.outputs.TAG_FREQAI_RL }}
117+
run: |
118+
docker build --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG} -t ${CACHE_IMAGE}:${TAG_PLOT} -f docker/Dockerfile.plot .
119+
docker build --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG} -t ${CACHE_IMAGE}:${TAG_FREQAI} -f docker/Dockerfile.freqai .
120+
docker build --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_FREQAI} -t ${CACHE_IMAGE}:${TAG_FREQAI_RL} -f docker/Dockerfile.freqai_rl .
121+
122+
123+
- name: Run backtest in Docker
124+
env:
125+
TAG: ${{ steps.tags.outputs.TAG }}
126+
run: |
127+
docker run --rm -v $(pwd)/tests/testdata/config.tests.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests ${CACHE_IMAGE}:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3
128+
129+
- name: Push cache images
130+
env:
131+
TAG: ${{ steps.tags.outputs.TAG }}
132+
TAG_PLOT: ${{ steps.tags.outputs.TAG_PLOT }}
133+
TAG_FREQAI: ${{ steps.tags.outputs.TAG_FREQAI }}
134+
TAG_FREQAI_RL: ${{ steps.tags.outputs.TAG_FREQAI_RL }}
77135
run: |
78-
build_helpers/publish_docker_multi.sh
136+
docker push ${CACHE_IMAGE}:$TAG
137+
docker push ${CACHE_IMAGE}:$TAG_PLOT
138+
docker push ${CACHE_IMAGE}:$TAG_FREQAI
139+
docker push ${CACHE_IMAGE}:$TAG_FREQAI_RL
140+
141+
- name: list Images
142+
run: |
143+
docker images
79144
80145
deploy-arm:
81146
name: "Deploy Docker ARM64"
@@ -91,31 +156,135 @@ jobs:
91156
with:
92157
persist-credentials: false
93158

94-
- name: Extract branch name
95-
env:
96-
BRANCH_NAME_INPUT: ${{ github.event.inputs.branch_name }}
97-
run: |
98-
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
99-
BRANCH_NAME="${BRANCH_NAME_INPUT}"
100-
else
101-
BRANCH_NAME="${GITHUB_REF##*/}"
102-
fi
103-
echo "GITHUB_REF='${GITHUB_REF}'"
104-
echo "BRANCH_NAME='${BRANCH_NAME}'"
105-
echo "BRANCH_NAME=${BRANCH_NAME}" >> "$GITHUB_ENV"
159+
- name: Set docker tag names
160+
id: tags
161+
uses: ./.github/actions/docker-tags
106162

107163
- name: Login to Docker Hub
108164
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
109165
with:
110166
username: ${{ secrets.DOCKER_USERNAME }}
111167
password: ${{ secrets.DOCKER_PASSWORD }}
112168

113-
- name: Build and test and push docker images
169+
- name: Login to github
170+
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
171+
with:
172+
registry: ghcr.io
173+
username: ${{ github.actor }}
174+
password: ${{ secrets.GITHUB_TOKEN }}
175+
176+
- name: Build image without cache
177+
if: github.event_name == 'schedule'
114178
env:
115-
GHCR_USERNAME: ${{ github.actor }}
116-
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
179+
TAG_ARM: ${{ steps.tags.outputs.TAG_ARM }}
180+
run: |
181+
docker build -t ${IMAGE_NAME}:${TAG_ARM} .
182+
183+
- name: Build image with cache
184+
if: github.event_name != 'schedule'
185+
env:
186+
TAG_ARM: ${{ steps.tags.outputs.TAG_ARM }}
187+
run: |
188+
docker pull ${IMAGE_NAME}:${TAG_ARM} || true
189+
docker build --cache-from ${IMAGE_NAME}:${TAG_ARM} -t ${CACHE_IMAGE}:${TAG_ARM} .
190+
191+
- name: Run build for AI images
192+
env:
193+
TAG_ARM: ${{ steps.tags.outputs.TAG_ARM }}
194+
TAG_PLOT_ARM: ${{ steps.tags.outputs.TAG_PLOT_ARM }}
195+
TAG_FREQAI_ARM: ${{ steps.tags.outputs.TAG_FREQAI_ARM }}
196+
TAG_FREQAI_RL_ARM: ${{ steps.tags.outputs.TAG_FREQAI_RL_ARM }}
197+
run: |
198+
docker build --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_ARM} -t ${CACHE_IMAGE}:${TAG_PLOT_ARM} -f docker/Dockerfile.plot .
199+
docker build --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_ARM} -t ${CACHE_IMAGE}:${TAG_FREQAI_ARM} -f docker/Dockerfile.freqai .
200+
docker build --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_FREQAI_ARM} -t ${CACHE_IMAGE}:${TAG_FREQAI_RL_ARM} -f docker/Dockerfile.freqai_rl .
201+
202+
203+
- name: Run backtest in Docker
204+
env:
205+
TAG_ARM: ${{ steps.tags.outputs.TAG_ARM }}
206+
run: |
207+
docker run --rm -v $(pwd)/tests/testdata/config.tests.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests ${CACHE_IMAGE}:${TAG_ARM} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3
208+
209+
- name: Docker images
210+
run: |
211+
docker images
212+
213+
- name: Push cache images
214+
env:
215+
TAG_ARM: ${{ steps.tags.outputs.TAG_ARM }}
216+
TAG_PLOT_ARM: ${{ steps.tags.outputs.TAG_PLOT_ARM }}
217+
TAG_FREQAI_ARM: ${{ steps.tags.outputs.TAG_FREQAI_ARM }}
218+
TAG_FREQAI_RL_ARM: ${{ steps.tags.outputs.TAG_FREQAI_RL_ARM }}
219+
run: |
220+
docker push ${CACHE_IMAGE}:$TAG_PLOT_ARM
221+
docker push ${CACHE_IMAGE}:$TAG_FREQAI_ARM
222+
docker push ${CACHE_IMAGE}:$TAG_FREQAI_RL_ARM
223+
docker push ${CACHE_IMAGE}:$TAG_ARM
224+
225+
- name: Create manifests
226+
env:
227+
TAG_ARM: ${{ steps.tags.outputs.TAG_ARM }}
228+
TAG: ${{ steps.tags.outputs.TAG }}
229+
TAG_PI: ${{ steps.tags.outputs.TAG_PI }}
230+
run: |
231+
docker buildx imagetools create \
232+
--tag ${IMAGE_NAME}:${TAG} \
233+
--tag ${GHCR_IMAGE_NAME}:${TAG} \
234+
${CACHE_IMAGE}:${TAG} ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI}
235+
236+
- name: Create multiarch image - Plot
237+
env:
238+
TAG_PLOT: ${{ steps.tags.outputs.TAG_PLOT }}
239+
TAG_PLOT_ARM: ${{ steps.tags.outputs.TAG_PLOT_ARM }}
240+
run: |
241+
docker buildx imagetools create \
242+
--tag ${IMAGE_NAME}:${TAG_PLOT} \
243+
--tag ${GHCR_IMAGE_NAME}:${TAG_PLOT} \
244+
${CACHE_IMAGE}:${TAG_PLOT} ${CACHE_IMAGE}:${TAG_PLOT_ARM}
245+
246+
- name: Create multiarch image - FreqAI
247+
env:
248+
TAG_FREQAI: ${{ steps.tags.outputs.TAG_FREQAI }}
249+
TAG_FREQAI_ARM: ${{ steps.tags.outputs.TAG_FREQAI_ARM }}
250+
run: |
251+
docker buildx imagetools create \
252+
--tag ${IMAGE_NAME}:${TAG_FREQAI} \
253+
--tag ${GHCR_IMAGE_NAME}:${TAG_FREQAI} \
254+
${CACHE_IMAGE}:${TAG_FREQAI} ${CACHE_IMAGE}:${TAG_FREQAI_ARM}
255+
256+
- name: Create multiarch image - FreqAI RL
257+
env:
258+
TAG_FREQAI_RL: ${{ steps.tags.outputs.TAG_FREQAI_RL }}
259+
TAG_FREQAI_RL_ARM: ${{ steps.tags.outputs.TAG_FREQAI_RL_ARM }}
260+
TAG_FREQAI_TORCH: ${{ steps.tags.outputs.TAG_FREQAI_TORCH }}
261+
run: |
262+
# Create special Torch tag - which is identical to the RL tag.
263+
docker buildx imagetools create \
264+
--tag ${IMAGE_NAME}:${TAG_FREQAI_RL} \
265+
--tag ${GHCR_IMAGE_NAME}:${TAG_FREQAI_RL} \
266+
--tag ${IMAGE_NAME}:${TAG_FREQAI_TORCH} \
267+
--tag ${GHCR_IMAGE_NAME}:${TAG_FREQAI_TORCH} \
268+
${CACHE_IMAGE}:${TAG_FREQAI_RL} ${CACHE_IMAGE}:${TAG_FREQAI_RL_ARM}
269+
270+
- name: Tag latest
271+
if: env.TAG == 'develop'
272+
env:
273+
TAG: ${{ steps.tags.outputs.TAG }}
274+
run: |
275+
# Tag image as latest
276+
docker buildx imagetools create \
277+
--tag ${GHCR_IMAGE_NAME}:${TAG} \
278+
--tag ${GHCR_IMAGE_NAME}:latest \
279+
${IMAGE_NAME}:${TAG}
280+
281+
- name: Docker images
282+
run: |
283+
docker images
284+
285+
- name: Image cleanup
117286
run: |
118-
build_helpers/publish_docker_arm64.sh
287+
docker image prune -a --force --filter "until=24h"
119288
120289
- name: Discord notification
121290
uses: rjstone/discord-webhook-notify@c2597273488aeda841dd1e891321952b51f7996f #v2.2.1

0 commit comments

Comments
 (0)