Skip to content

Commit 9b593f6

Browse files
aiakideaiakide
authored
Add dagster and mlflow docker images (#3)
* feat: Add dagster Dockerfile. Adjust project structure * ci: Add dagster image build step * feat: Add mlflow Dockerfile. Adjust jupyter and dagster Dockerfile * ci: Add mlflow Docker image build step * ci: fix invalid value `contest` * ci: Make ids unique * feat: Add ´torch` and `torchvision` to jupyter image * feat: Add seaborn as dependency
1 parent 1d974db commit 9b593f6

File tree

10 files changed

+1645
-1045
lines changed

10 files changed

+1645
-1045
lines changed

‎.github/workflows/main.yml‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,17 +48,42 @@ jobs:
4848
restore-keys: |
4949
${{ runner.os }}-buildx-
5050
51-
- name: Build and push Docker image
52-
id: docker_build_amd64
51+
- name: Build and push jupyter Docker image
52+
id: docker_build_amd64_jupyter
5353
uses: docker/[email protected]
5454
with:
5555
context: .
56-
tags: codecentric/from-jupyter-to-production-baseimage:latest
56+
file: ./docker/jupyter/Dockerfile
57+
tags: codecentric/from-jupyter-to-production-jupyter:latest
5758
push: true
5859
builder: ${{ steps.buildx.outputs.name }}
5960
cache-from: type=local,src=/tmp/.buildx-cache
6061
cache-to: type=local,dest=/tmp/.buildx-cache
6162
platforms: linux/amd64,linux/arm64
62-
63+
- name: Build and push dagster Docker image
64+
id: docker_build_amd64_dagster
65+
uses: docker/[email protected]
66+
with:
67+
context: .
68+
file: ./docker/dagster/Dockerfile
69+
tags: codecentric/from-jupyter-to-production-dagster:latest
70+
push: true
71+
builder: ${{ steps.buildx.outputs.name }}
72+
cache-from: type=local,src=/tmp/.buildx-cache
73+
cache-to: type=local,dest=/tmp/.buildx-cache
74+
platforms: linux/amd64,linux/arm64
75+
- name: Build and push mlflow Docker image
76+
id: docker_build_amd64_mlflow
77+
uses: docker/[email protected]
78+
with:
79+
context: .
80+
file: ./docker/mlflow/Dockerfile
81+
tags: codecentric/from-jupyter-to-production-mlflow:latest
82+
push: true
83+
builder: ${{ steps.buildx.outputs.name }}
84+
cache-from: type=local,src=/tmp/.buildx-cache
85+
cache-to: type=local,dest=/tmp/.buildx-cache
86+
platforms: linux/amd64,linux/arm64
87+
6388
- name: Image digest
6489
run: echo ${{ steps.docker_build.outputs.digest }}

‎configs/jupyter_lab_config.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -778,6 +778,7 @@
778778
# Default: 'jupyter_server.services.contents.largefilemanager.AsyncLargeFileManager'
779779
# c.ServerApp.contents_manager_class = 'jupyter_server.services.contents.largefilemanager.AsyncLargeFileManager'
780780

781+
781782
## DEPRECATED. Use IdentityProvider.cookie_options
782783
# Default: {}
783784
# c.ServerApp.cookie_options = {}
@@ -896,7 +897,7 @@
896897
# be used to enable and disable the loading ofthe extensions. The extensions
897898
# will be loaded in alphabetical order.
898899
# Default: {}
899-
# c.ServerApp.jpserver_extensions = {}
900+
c.ServerApp.jpserver_extensions = {"jupytext":True}
900901

901902
## The kernel manager class to use.
902903
# Default: 'jupyter_server.services.kernels.kernelmanager.MappingKernelManager'

‎configs/jupytext.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[formats]
2+
"dagster/" = "ipynb"
3+
"dagster/scripts/" = "py:percent"

‎docker/dagster/Dockerfile‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
# syntax=docker/dockerfile:1
2+
3+
# Based on https://gist.githubusercontent.com/usr-ein/c42d98abca3cb4632ab0c2c6aff8c88a/raw/19dcc899f68d0b08c2c137d3fd01715b0c84bac9/Dockerfile
4+
5+
################################
6+
# PYTHON-BASE
7+
# Sets up all our shared environment variables
8+
################################
9+
FROM --platform=$TARGETPLATFORM python:3.11-slim as python-base
10+
11+
ARG TARGETPLATFORM
12+
13+
# python
14+
ENV PYTHONUNBUFFERED=1 \
15+
# prevents python creating .pyc files
16+
PYTHONDONTWRITEBYTECODE=1 \
17+
\
18+
# pip
19+
PIP_DISABLE_PIP_VERSION_CHECK=on \
20+
PIP_DEFAULT_TIMEOUT=100 \
21+
\
22+
# poetry
23+
# https://python-poetry.org/docs/configuration/#using-environment-variables
24+
POETRY_VERSION=1.6.1 \
25+
# make poetry install to this location
26+
POETRY_HOME="/opt/poetry" \
27+
# make poetry create the virtual environment in the project's root
28+
# it gets named `.venv`
29+
POETRY_VIRTUALENVS_IN_PROJECT=true \
30+
# do not ask any interactive question
31+
POETRY_NO_INTERACTION=1 \
32+
\
33+
# paths
34+
# this is where our requirements + virtual environment will live
35+
PYSETUP_PATH="/opt/pysetup" \
36+
VENV_PATH="/opt/pysetup/.venv"
37+
38+
39+
# prepend poetry and venv to path
40+
ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
41+
42+
43+
################################
44+
# BUILDER-BASE
45+
# Used to build deps + create our virtual environment
46+
################################
47+
FROM python-base as builder-base
48+
RUN apt-get update \
49+
&& apt-get install --no-install-recommends -y \
50+
# deps for installing poetry
51+
curl \
52+
# deps for building python deps
53+
build-essential
54+
55+
# install poetry - respects $POETRY_VERSION & $POETRY_HOME
56+
# The --mount will mount the buildx cache directory to where
57+
# Poetry and Pip store their cache so that they can re-use it
58+
RUN --mount=type=cache,target=/root/.cache \
59+
curl -sSL https://install.python-poetry.org | python3 -
60+
61+
# copy project requirement files here to ensure they will be cached.
62+
WORKDIR $PYSETUP_PATH
63+
COPY ../../poetry.lock pyproject.toml ./
64+
65+
# install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
66+
RUN --mount=type=cache,target=/root/.cache \
67+
poetry install --with dagster-webserver,dagster --without mlflow
68+
69+
70+
################################
71+
# PRODUCTION
72+
# Final image used for runtime
73+
################################
74+
FROM python-base as production
75+
COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
76+
77+
RUN mkdir -p /opt/dagster/dagster_home /opt/dagster/app
78+
79+
ENV DAGSTER_HOME=/opt/dagster/dagster_home/
80+
81+
# Copy your workspace to /opt/dagster/app
82+
83+
COPY ./docker/dagster/workspace.yaml /opt/dagster/app/
84+
COPY ./docker/dagster/dagster.yaml $DAGSTER_HOME/
85+
86+
87+
WORKDIR /opt/dagster/app
88+
EXPOSE 3000
89+
ENV SHELL="/bin/bash"
90+
91+
ENTRYPOINT ["dagster","dev","-h","0.0.0.0","-p","3000"]

‎docker/dagster/dagster.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
telemetry:
2+
enabled: False

‎docker/dagster/workspace.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# workspace.yaml
2+
3+
load_from:
4+
- python_file: /opt/dagster/code/dagster-demo.py

‎docker/jupyter/Dockerfile‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# syntax=docker/dockerfile:1
2+
3+
# Based on https://gist.githubusercontent.com/usr-ein/c42d98abca3cb4632ab0c2c6aff8c88a/raw/19dcc899f68d0b08c2c137d3fd01715b0c84bac9/Dockerfile
4+
5+
################################
6+
# PYTHON-BASE
7+
# Sets up all our shared environment variables
8+
################################
9+
FROM --platform=$TARGETPLATFORM python:3.11-slim as python-base
10+
11+
ARG TARGETPLATFORM
12+
13+
# python
14+
ENV PYTHONUNBUFFERED=1 \
15+
# prevents python creating .pyc files
16+
PYTHONDONTWRITEBYTECODE=1 \
17+
\
18+
# pip
19+
PIP_DISABLE_PIP_VERSION_CHECK=on \
20+
PIP_DEFAULT_TIMEOUT=100 \
21+
\
22+
# poetry
23+
# https://python-poetry.org/docs/configuration/#using-environment-variables
24+
POETRY_VERSION=1.6.1 \
25+
# make poetry install to this location
26+
POETRY_HOME="/opt/poetry" \
27+
# make poetry create the virtual environment in the project's root
28+
# it gets named `.venv`
29+
POETRY_VIRTUALENVS_IN_PROJECT=true \
30+
# do not ask any interactive question
31+
POETRY_NO_INTERACTION=1 \
32+
\
33+
# paths
34+
# this is where our requirements + virtual environment will live
35+
PYSETUP_PATH="/opt/pysetup" \
36+
VENV_PATH="/opt/pysetup/.venv"
37+
38+
39+
# prepend poetry and venv to path
40+
ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
41+
42+
43+
################################
44+
# BUILDER-BASE
45+
# Used to build deps + create our virtual environment
46+
################################
47+
FROM python-base as builder-base
48+
RUN apt-get update \
49+
&& apt-get install --no-install-recommends -y \
50+
# deps for installing poetry
51+
curl \
52+
# deps for building python deps
53+
build-essential \
54+
# git
55+
git
56+
57+
# install poetry - respects $POETRY_VERSION & $POETRY_HOME
58+
# The --mount will mount the buildx cache directory to where
59+
# Poetry and Pip store their cache so that they can re-use it
60+
RUN --mount=type=cache,target=/root/.cache \
61+
curl -sSL https://install.python-poetry.org | python3 -
62+
63+
# copy project requirement files here to ensure they will be cached.
64+
WORKDIR $PYSETUP_PATH
65+
COPY ../../poetry.lock pyproject.toml ./
66+
67+
# install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
68+
RUN --mount=type=cache,target=/root/.cache \
69+
poetry install --with dagster,mlflow --without dagster-webserver
70+
71+
################################
72+
# PRODUCTION
73+
# Final image used for runtime
74+
################################
75+
FROM python-base as production
76+
COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
77+
78+
ADD ../../configs/jupyter_lab_config.py /root/.jupyter/jupyter_lab_config.py
79+
ADD ../../configs/jupytext.toml /root/.config/jupytext.toml
80+
81+
WORKDIR /workshop
82+
EXPOSE 8888 4141
83+
ENV SHELL="/bin/bash"
84+
ENV GIT_PYTHON_REFRESH="quiet"
85+
86+
ENTRYPOINT ["jupyter", "lab"]

‎Dockerfile‎ renamed to ‎docker/mlflow/Dockerfile‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -50,21 +50,23 @@ RUN apt-get update \
5050
# deps for installing poetry
5151
curl \
5252
# deps for building python deps
53-
build-essential
53+
build-essential \
54+
gcc \
55+
git
5456

5557
# install poetry - respects $POETRY_VERSION & $POETRY_HOME
56-
# The --mount will mount the buildx cache directory to where
58+
# The --mount will mount the buildx cache directory to where
5759
# Poetry and Pip store their cache so that they can re-use it
5860
RUN --mount=type=cache,target=/root/.cache \
5961
curl -sSL https://install.python-poetry.org | python3 -
6062

6163
# copy project requirement files here to ensure they will be cached.
6264
WORKDIR $PYSETUP_PATH
63-
COPY poetry.lock pyproject.toml ./
65+
COPY ../../poetry.lock pyproject.toml ./
6466

6567
# install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
6668
RUN --mount=type=cache,target=/root/.cache \
67-
poetry install
69+
poetry install --only mlflow
6870

6971

7072
################################
@@ -74,9 +76,11 @@ RUN --mount=type=cache,target=/root/.cache \
7476
FROM python-base as production
7577
COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
7678

77-
ADD configs/jupyter_lab_config.py /root/.jupyter/jupyter_lab_config.py
78-
WORKDIR /workshop
79-
EXPOSE 8888 4141 5001
79+
80+
WORKDIR /ml_data
81+
EXPOSE 5001
8082
ENV SHELL="/bin/bash"
83+
ENV GIT_PYTHON_REFRESH="quiet"
84+
8185

82-
ENTRYPOINT ["jupyter", "lab"]
86+
ENTRYPOINT ["mlflow","server","--host","0.0.0.0","--port","5001"]

0 commit comments

Comments
 (0)