Skip to content

Commit 64630bb

Browse files
committed
Merge branch 'feat/rebuild-pipeline'
2 parents 2c62edc + 2233dd2 commit 64630bb

File tree

7 files changed

+610
-333
lines changed

7 files changed

+610
-333
lines changed
Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
name: "Shared buildsteps"
2+
inputs:
3+
dockerhub_user:
4+
required: true
5+
dockerhub_password:
6+
required: true
7+
runner_os:
8+
required: true
9+
github_sha:
10+
required: true
11+
runs:
12+
using: "composite"
13+
steps:
14+
- name: Login to Docker Hub 🔐🐳
15+
uses: docker/login-action@v1
16+
with:
17+
username: ${{ inputs.dockerhub_user }}
18+
password: ${{ inputs.dockerhub_password }}
19+
20+
- name: Set up QEMU 🏗️
21+
uses: docker/setup-qemu-action@v1
22+
with:
23+
platforms: amd64,arm64
24+
25+
- name: Free disk space 🧹
26+
shell: bash
27+
run: |
28+
sudo swapoff -a
29+
sudo rm -f /swapfile
30+
sudo apt clean
31+
docker rmi $(docker image ls -aq)
32+
df -h
33+
34+
- name: Set up Docker Buildx 🏗️
35+
id: buildx
36+
uses: docker/setup-buildx-action@v1
37+
38+
- name: Cache Docker layers 💾
39+
uses: actions/cache@v2
40+
with:
41+
path: /tmp/.buildx-cache
42+
key: ${{ inputs.runner_os }}-buildx-${{ inputs.github_sha }}
43+
restore-keys: |
44+
${{ inputs.runner_os }}-buildx-

.github/workflows/main.yml

Lines changed: 56 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,89 +1,86 @@
1-
# This is a basic workflow to help you get started with Actions
1+
#This GitHub Workflow builds the necessary docker images for the `from jupyter to production workshop`
2+
name: Build and publish workshop Docker images 🔨🐳🚀
23

3-
name: CI
4-
5-
# Controls when the action will run.
64
on:
7-
# Triggers the workflow on push or pull request events but only for the master branch
5+
# Triggers the workflow when a new tag with the pattern 'v*.*.*' is pushed to the repository
86
push:
9-
branches: [ master ]
10-
pull_request:
11-
branches: [ master ]
7+
tags:
8+
- 'v*.*.*'
129

1310
# Allows you to run this workflow manually from the Actions tab
1411
workflow_dispatch:
1512

16-
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
1713
jobs:
18-
# This workflow contains a single job called "build"
19-
build:
20-
# The type of runner that the job will run on
14+
# This job creates and publishes the mlflow docker images
15+
build-mlflow:
2116
runs-on: ubuntu-latest
22-
23-
# Steps represent a sequence of tasks that will be executed as part of the job
2417
steps:
25-
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
26-
- uses: actions/checkout@v2
27-
28-
- name: Login to Docker Hub
29-
uses: docker/login-action@v1
30-
with:
31-
username: ${{ secrets.DOCKERHUB_USER }}
32-
password: ${{ secrets.DOCKERHUB_PASSWORD }}
18+
- name: Checkout Repo ✅
19+
uses: actions/checkout@v2
3320

34-
- name: Set up QEMU
35-
uses: docker/setup-qemu-action@v1
21+
- name: Job preparation 🛫 # Steps that must be carried out before creating the Docker images
22+
uses: ./.github/actions/shared-build-steps
3623
with:
37-
platforms: amd64
24+
dockerhub_user: ${{ secrets.DOCKERHUB_USER }}
25+
dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }}
26+
runner_os: ${{ runner.os }}
27+
github_sha: ${{ github.sha }}
3828

39-
- name: Set up Docker Buildx
40-
id: buildx
41-
uses: docker/setup-buildx-action@v1
42-
43-
- name: Cache Docker layers
44-
uses: actions/cache@v2
29+
- name: Docker meta 💬 # Creating meta information such as the image tag for the Docker image
30+
id: meta
31+
uses: docker/metadata-action@v5
4532
with:
46-
path: /tmp/.buildx-cache
47-
key: ${{ runner.os }}-buildx-${{ github.sha }}
48-
restore-keys: |
49-
${{ runner.os }}-buildx-
50-
51-
- name: Build and push jupyter Docker image
52-
id: docker_build_amd64_jupyter
33+
images: codecentric/from-jupyter-to-production-mlflow
34+
35+
- name: Build and push mlflow Docker image 🔨🐳
36+
id: docker_build_mlflow
5337
uses: docker/[email protected]
5438
with:
5539
context: .
56-
file: ./docker/jupyter/Dockerfile
57-
tags: codecentric/from-jupyter-to-production-jupyter:latest
58-
push: true
40+
file: ./docker/mlflow/Dockerfile
41+
tags: ${{ steps.meta.outputs.tags }}
42+
push: ${{ github.event_name != 'pull_request' }}
5943
builder: ${{ steps.buildx.outputs.name }}
6044
cache-from: type=local,src=/tmp/.buildx-cache
6145
cache-to: type=local,dest=/tmp/.buildx-cache
62-
platforms: linux/amd64
63-
- name: Build and push dagster Docker image
64-
id: docker_build_amd64_dagster
65-
uses: docker/[email protected]
46+
platforms: linux/amd64,linux/arm64
47+
48+
- name: Image digest
49+
run: echo ${{ steps.docker_build.outputs.digest }}
50+
51+
# This job creates and publishes the jupyter docker images
52+
build-jupyter:
53+
runs-on: ubuntu-latest
54+
steps:
55+
- name: Checkout Repo ✅
56+
uses: actions/checkout@v2
57+
58+
- name: Job preparation 🛫 # Steps that must be carried out before creating the Docker images
59+
uses: ./.github/actions/shared-build-steps
6660
with:
67-
context: .
68-
file: ./docker/dagster/Dockerfile
69-
tags: codecentric/from-jupyter-to-production-dagster:latest
70-
push: true
71-
builder: ${{ steps.buildx.outputs.name }}
72-
cache-from: type=local,src=/tmp/.buildx-cache
73-
cache-to: type=local,dest=/tmp/.buildx-cache
74-
platforms: linux/amd64
75-
- name: Build and push mlflow Docker image
76-
id: docker_build_amd64_mlflow
61+
dockerhub_user: ${{ secrets.DOCKERHUB_USER }}
62+
dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }}
63+
runner_os: ${{ runner.os }}
64+
github_sha: ${{ github.sha }}
65+
66+
- name: Docker meta 💬 # Creating meta information such as the image tag for the Docker image
67+
id: meta
68+
uses: docker/metadata-action@v5
69+
with:
70+
images: codecentric/from-jupyter-to-production-jupyter
71+
72+
- name: Build and push juypter Docker image 🔨🐳
73+
id: docker_build_mlflow
7774
uses: docker/[email protected]
7875
with:
7976
context: .
80-
file: ./docker/mlflow/Dockerfile
81-
tags: codecentric/from-jupyter-to-production-mlflow:latest
82-
push: true
77+
file: ./docker/jupyter/Dockerfile
78+
tags: ${{ steps.meta.outputs.tags }}
79+
push: ${{ github.event_name != 'pull_request' }}
8380
builder: ${{ steps.buildx.outputs.name }}
8481
cache-from: type=local,src=/tmp/.buildx-cache
8582
cache-to: type=local,dest=/tmp/.buildx-cache
86-
platforms: linux/amd64
83+
platforms: linux/amd64,linux/arm64
8784

8885
- name: Image digest
8986
run: echo ${{ steps.docker_build.outputs.digest }}

.pre-commit-config.yaml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
repos:
2+
- repo: https://github.com/pre-commit/pre-commit-hooks
3+
rev: v2.3.0
4+
hooks:
5+
- id: check-yaml
6+
- id: end-of-file-fixer
7+
- id: trailing-whitespace
8+
- repo: https://github.com/psf/black
9+
rev: 22.10.0
10+
hooks:
11+
- id: black

docker/jupyter/Dockerfile

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,10 @@ RUN apt-get update \
5050
# deps for installing poetry
5151
curl \
5252
# deps for building python deps
53-
build-essential \
54-
# git
55-
git
53+
build-essential
5654

5755
# install poetry - respects $POETRY_VERSION & $POETRY_HOME
58-
# The --mount will mount the buildx cache directory to where
56+
# The --mount will mount the buildx cache directory to where
5957
# Poetry and Pip store their cache so that they can re-use it
6058
RUN --mount=type=cache,target=/root/.cache \
6159
curl -sSL https://install.python-poetry.org | python3 -
@@ -76,12 +74,13 @@ RUN --mount=type=cache,target=/root/.cache \
7674
FROM python-base as production
7775
COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
7876

77+
RUN apt-get update && apt-get install -y git
78+
7979
ADD ../../configs/jupyter_lab_config.py /root/.jupyter/jupyter_lab_config.py
8080
ADD ../../configs/jupytext.toml /root/.config/jupytext.toml
8181

8282
WORKDIR /workshop
8383
EXPOSE 8888 4141
8484
ENV SHELL="/bin/bash"
85-
ENV GIT_PYTHON_REFRESH="quiet"
8685

8786
ENTRYPOINT ["jupyter", "lab"]

docker/mlflow/Dockerfile

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,7 @@ RUN apt-get update \
5151
curl \
5252
# deps for building python deps
5353
build-essential \
54-
gcc \
55-
git
54+
gcc
5655

5756
# install poetry - respects $POETRY_VERSION & $POETRY_HOME
5857
# The --mount will mount the buildx cache directory to where
@@ -76,11 +75,9 @@ RUN --mount=type=cache,target=/root/.cache \
7675
FROM python-base as production
7776
COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
7877

79-
8078
WORKDIR /ml_data
8179
EXPOSE 5001
8280
ENV SHELL="/bin/bash"
83-
ENV GIT_PYTHON_REFRESH="quiet"
8481

8582

8683
ENTRYPOINT ["mlflow","server","--host","0.0.0.0","--port","5001"]

0 commit comments

Comments
 (0)