diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4b0fed2..42e2dd3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,13 +23,13 @@ jobs: with: imagename: moonvision/${{ env.IMAGE_PRE }}python-base path: docker/python-base - dockerfile: docker/python-base/Dockerfile + dockerfile: docker/python-base/DockerfileMicromamba baseimage: mambaorg/micromamba:1 - uses: ./.github/create-docker with: imagename: moonvision/${{ env.IMAGE_PRE }}python-base path: docker/python-base - dockerfile: docker/python-base/Dockerfile + dockerfile: docker/python-base/DockerfileMicromamba baseimage: mambaorg/micromamba:1 additionaltag: 3.11 additional_buildargs: env_file=environment_3.11.yml @@ -41,17 +41,25 @@ jobs: with: imagename: moonvision/${{ env.IMAGE_PRE }}python-base path: docker/python-base - dockerfile: docker/python-base/DockerfileUV + dockerfile: docker/python-base/Dockerfile additionaltag: uv additional_buildargs: python_version=3.8 - uses: ./.github/create-docker with: imagename: moonvision/${{ env.IMAGE_PRE }}python-base path: docker/python-base - dockerfile: docker/python-base/DockerfileUV + dockerfile: docker/python-base/Dockerfile additionaltag: uv-3.11 additional_buildargs: |- python_version=3.11 + - uses: ./.github/create-docker + with: + imagename: moonvision/${{ env.IMAGE_PRE }}python-base + path: docker/python-base + dockerfile: docker/python-base/Dockerfile + additionaltag: uv-3.13 + additional_buildargs: |- + python_version=3.13 build-ffmpeg: runs-on: ubuntu-latest steps: @@ -193,7 +201,7 @@ jobs: imagename: moonvision/${{ env.IMAGE_PRE }}moonbox additionaltag: mini path: docker/moonbox - dockerfile: docker/moonbox/Dockerfile + dockerfile: docker/moonbox/DockerfileMicromamba baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:latest additional_buildargs: |- ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-4.2.1 @@ -211,7 +219,7 @@ jobs: imagename: moonvision/${{ env.IMAGE_PRE }}moonbox additionaltag: mini-3.11 path: docker/moonbox - dockerfile: docker/moonbox/Dockerfile + dockerfile: docker/moonbox/DockerfileMicromamba baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:3.11-latest additional_buildargs: |- ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-4.2.1 @@ -229,7 +237,7 @@ jobs: imagename: moonvision/${{ env.IMAGE_PRE }}moonbox additionaltag: cuda path: docker/moonbox - dockerfile: docker/moonbox/Dockerfile + dockerfile: docker/moonbox/DockerfileMicromamba baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:latest additional_buildargs: |- with_cuda=true @@ -248,7 +256,7 @@ jobs: imagename: moonvision/${{ env.IMAGE_PRE }}moonbox additionaltag: cuda-3.11 path: docker/moonbox - dockerfile: docker/moonbox/Dockerfile + dockerfile: docker/moonbox/DockerfileMicromamba baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:3.11-latest additional_buildargs: |- with_cuda=true @@ -266,7 +274,7 @@ jobs: imagename: moonvision/${{ env.IMAGE_PRE }}moonbox additionaltag: mini-uv path: docker/moonbox - dockerfile: docker/moonbox/DockerfileUV + dockerfile: docker/moonbox/Dockerfile baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:uv-latest additional_buildargs: |- ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-4.2.1 @@ -282,10 +290,26 @@ jobs: imagename: moonvision/${{ env.IMAGE_PRE }}moonbox additionaltag: mini-uv-3.11 path: docker/moonbox - dockerfile: docker/moonbox/DockerfileUV + dockerfile: docker/moonbox/Dockerfile baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:uv-3.11-latest additional_buildargs: |- ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-4.2.1 + docker-moonbox-mini-uv-313: + needs: + - docker-python-base-uv + - build-ffmpeg + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/create-docker + with: + imagename: moonvision/${{ env.IMAGE_PRE }}moonbox + additionaltag: mini-uv-3.13 + path: docker/moonbox + dockerfile: docker/moonbox/Dockerfile + baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:uv-3.13-latest + additional_buildargs: |- + ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-4.2.1 docker-moonbox-cuda-uv: needs: - docker-python-base-uv @@ -298,7 +322,7 @@ jobs: imagename: moonvision/${{ env.IMAGE_PRE }}moonbox additionaltag: cuda-uv path: docker/moonbox - dockerfile: docker/moonbox/DockerfileUV + dockerfile: docker/moonbox/Dockerfile baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:uv-latest additional_buildargs: |- with_cuda=true @@ -315,8 +339,25 @@ jobs: imagename: moonvision/${{ env.IMAGE_PRE }}moonbox additionaltag: cuda-uv-3.11 path: docker/moonbox - dockerfile: docker/moonbox/DockerfileUV + dockerfile: docker/moonbox/Dockerfile baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:uv-3.11-latest additional_buildargs: |- with_cuda=true ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-cuda-4.2.1 + docker-moonbox-cuda-uv-313: + needs: + - docker-python-base-uv + - build-ffmpeg-cuda + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/create-docker + with: + imagename: moonvision/${{ env.IMAGE_PRE }}moonbox + additionaltag: cuda-uv-3.13 + path: docker/moonbox + dockerfile: docker/moonbox/Dockerfile + baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:uv-3.13-latest + additional_buildargs: |- + with_cuda=true + ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-cuda-4.2.1 diff --git a/docker/moonbox/Dockerfile b/docker/moonbox/Dockerfile index 131b0da..ee04578 100644 --- a/docker/moonbox/Dockerfile +++ b/docker/moonbox/Dockerfile @@ -1,20 +1,22 @@ -ARG baseimage='moonvision/python-base' +ARG baseimage='moonvision/python-base:uv-latest' ARG ffmpeg_from_docker='moonvision/custom-builds:ffmpeg-4.2.1' -ARG pytorch_from_docker='moonvision/custom-builds:pytorch-1.11.0_torchvision-0.12.0' FROM $ffmpeg_from_docker as ffmpeg_builder -FROM $pytorch_from_docker as pytorch_builder FROM ${baseimage} as installer ARG with_cuda='false' +ARG DEBIAN_FRONTEND='noninteractive' + # fixme: Why should gcc be installed both here and in python-base? And the same for checkinstall. +RUN apt-get update --yes --no-install-recommends \ + && apt-get install --yes gcc g++ checkinstall cmake --no-install-recommends + COPY --from=ffmpeg_builder /packages /ffmpeg-packages -COPY --from=pytorch_builder /packages /pytorch-packages COPY . /bd_build ARG MAMBA_DOCKERFILE_ACTIVATE=1 -RUN with_cuda=$with_cuda bash /bd_build/install_deps.sh +RUN with_cuda=$with_cuda bash /bd_build/install_deps_uv.sh # squash /packages FROM ${baseimage} diff --git a/docker/moonbox/DockerfileUV b/docker/moonbox/DockerfileMicromamba similarity index 63% rename from docker/moonbox/DockerfileUV rename to docker/moonbox/DockerfileMicromamba index ee04578..131b0da 100644 --- a/docker/moonbox/DockerfileUV +++ b/docker/moonbox/DockerfileMicromamba @@ -1,22 +1,20 @@ -ARG baseimage='moonvision/python-base:uv-latest' +ARG baseimage='moonvision/python-base' ARG ffmpeg_from_docker='moonvision/custom-builds:ffmpeg-4.2.1' +ARG pytorch_from_docker='moonvision/custom-builds:pytorch-1.11.0_torchvision-0.12.0' FROM $ffmpeg_from_docker as ffmpeg_builder +FROM $pytorch_from_docker as pytorch_builder FROM ${baseimage} as installer ARG with_cuda='false' -ARG DEBIAN_FRONTEND='noninteractive' - # fixme: Why should gcc be installed both here and in python-base? And the same for checkinstall. -RUN apt-get update --yes --no-install-recommends \ - && apt-get install --yes gcc g++ checkinstall cmake --no-install-recommends - COPY --from=ffmpeg_builder /packages /ffmpeg-packages +COPY --from=pytorch_builder /packages /pytorch-packages COPY . /bd_build ARG MAMBA_DOCKERFILE_ACTIVATE=1 -RUN with_cuda=$with_cuda bash /bd_build/install_deps_uv.sh +RUN with_cuda=$with_cuda bash /bd_build/install_deps.sh # squash /packages FROM ${baseimage} diff --git a/docker/python-base/Dockerfile b/docker/python-base/Dockerfile index 35b4b14..75704e5 100644 --- a/docker/python-base/Dockerfile +++ b/docker/python-base/Dockerfile @@ -1,9 +1,18 @@ -ARG baseimage='mambaorg/micromamba:1.0.0' +ARG python_version +# This stage is copied from mambaorg/micromamba Dockerfile and contains everything except the actual micromamba stuff +FROM ghcr.io/astral-sh/uv:python${python_version}-bookworm-slim AS preparation_stage +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \ + && rm -rf /var/lib/apt/lists/* -# Build runit-docker -FROM ${baseimage} AS build_runit_docker +ARG CERT_SOURCE='/etc/ssl/certs/ca-certificates.crt' +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +ENV ENV_NAME="base" + +# Default command for "docker run" +CMD ["/bin/bash"] -USER root +# Build runit-docker +FROM preparation_stage AS build_runit_docker RUN apt-get update \ && apt-get install -y curl make gcc libc-dev checkinstall --no-install-recommends \ @@ -17,24 +26,9 @@ RUN cd /tmp \ && mkdir /packages \ && mv *.deb /packages -FROM ${baseimage} - -# Install python -COPY --chown=$MAMBA_USER:$MAMBA_USER . /home/$MAMBA_USER/bd_build -ARG env_file='environment.yml' -RUN micromamba install -y -f /home/$MAMBA_USER/bd_build/$env_file && \ - micromamba clean -yaf +FROM preparation_stage -# Manifest mamba python for exec environments -ENV CPATH "$MAMBA_ROOT_PREFIX/include" -ENV PATH "$MAMBA_ROOT_PREFIX/bin:$PATH" - -USER root -# You may seal production images by running: -#USER $MAMBA_USER - -# adduser template of $MAMBA_USER -RUN echo "SKEL=/home/$MAMBA_USER" >> /etc/default/useradd +SHELL ["/bin/bash", "-c"] # Install other tools ARG DEBIAN_FRONTEND=noninteractive @@ -48,6 +42,13 @@ RUN apt-get update -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# Configure uv +# Without this you will have to pass `--no-sync` for each `uv run` call, because it will try to install dev requirements otherwise +ENV UV_NO_SYNC=true +# Next two are needed, because otherwise uv cannot find certificates +ENV UV_NATIVE_TLS=true \ + SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + # Install runit-docker COPY --from=build_runit_docker /packages /packages RUN dpkg -i /packages/*.deb && rm -rf /packages diff --git a/docker/python-base/DockerfileMicromamba b/docker/python-base/DockerfileMicromamba new file mode 100644 index 0000000..35b4b14 --- /dev/null +++ b/docker/python-base/DockerfileMicromamba @@ -0,0 +1,54 @@ +ARG baseimage='mambaorg/micromamba:1.0.0' + +# Build runit-docker +FROM ${baseimage} AS build_runit_docker + +USER root + +RUN apt-get update \ + && apt-get install -y curl make gcc libc-dev checkinstall --no-install-recommends \ + && apt-get clean + +RUN cd /tmp \ + && curl -o runit-docker-1.2.tar.gz -L https://github.com/kosma/runit-docker/archive/refs/tags/1.2.tar.gz \ + && tar xf runit-docker-1.2.tar.gz \ + && cd runit-docker-1.2 \ + && checkinstall --install=no \ + && mkdir /packages \ + && mv *.deb /packages + +FROM ${baseimage} + +# Install python +COPY --chown=$MAMBA_USER:$MAMBA_USER . /home/$MAMBA_USER/bd_build +ARG env_file='environment.yml' +RUN micromamba install -y -f /home/$MAMBA_USER/bd_build/$env_file && \ + micromamba clean -yaf + +# Manifest mamba python for exec environments +ENV CPATH "$MAMBA_ROOT_PREFIX/include" +ENV PATH "$MAMBA_ROOT_PREFIX/bin:$PATH" + +USER root +# You may seal production images by running: +#USER $MAMBA_USER + +# adduser template of $MAMBA_USER +RUN echo "SKEL=/home/$MAMBA_USER" >> /etc/default/useradd + +# Install other tools +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends \ + tzdata \ + libglib2.0-0 \ + wget bzip2 curl git procps \ + runit \ + && apt-get purge --autoremove --yes python python3 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Install runit-docker +COPY --from=build_runit_docker /packages /packages +RUN dpkg -i /packages/*.deb && rm -rf /packages +CMD ["/usr/sbin/runit-docker"] diff --git a/docker/python-base/DockerfileUV b/docker/python-base/DockerfileUV deleted file mode 100644 index b5842f9..0000000 --- a/docker/python-base/DockerfileUV +++ /dev/null @@ -1,61 +0,0 @@ -ARG python_version -# This stage is copied from mambaorg/micromamba Dockerfile and contains everything except the actual micromamba stuff -FROM python:${python_version}-slim-bookworm AS preparation_stage -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -ARG CERT_SOURCE='/etc/ssl/certs/ca-certificates.crt' -ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 -ENV ENV_NAME="base" - -# Default command for "docker run" -CMD ["/bin/bash"] - -# Build runit-docker -FROM preparation_stage AS build_runit_docker - -RUN apt-get update \ - && apt-get install -y curl make gcc libc-dev checkinstall --no-install-recommends \ - && apt-get clean - -RUN cd /tmp \ - && curl -o runit-docker-1.2.tar.gz -L https://github.com/kosma/runit-docker/archive/refs/tags/1.2.tar.gz \ - && tar xf runit-docker-1.2.tar.gz \ - && cd runit-docker-1.2 \ - && checkinstall --install=no \ - && mkdir /packages \ - && mv *.deb /packages - -FROM preparation_stage - -SHELL ["/bin/bash", "-c"] - -# Install other tools -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update -y \ - && apt-get install -y --no-install-recommends \ - tzdata \ - libglib2.0-0 \ - wget bzip2 curl git procps \ - runit \ - && apt-get purge --autoremove --yes python python3 \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Install uv -ADD https://astral.sh/uv/install.sh /uv-installer.sh -RUN sh /uv-installer.sh && rm /uv-installer.sh -ENV PATH="/root/.local/bin/:$PATH" -# We use python base image so that we didn't have to run `uv run python` all the time -# This version should be preferred -ENV UV_PYTHON_PREFERENCE=system -# Without this you will have to pass `--no-sync` for each `uv run` call, because it will try to install dev requirements otherwise -ENV UV_NO_SYNC=true -# Next two are needed, because otherwise uv cannot find certificates -ENV UV_NATIVE_TLS=true -ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt - -# Install runit-docker -COPY --from=build_runit_docker /packages /packages -RUN dpkg -i /packages/*.deb && rm -rf /packages -CMD ["/usr/sbin/runit-docker"] diff --git a/docker/python-base/environment_uv.yml b/docker/python-base/environment_uv.yml deleted file mode 100644 index e4e961f..0000000 --- a/docker/python-base/environment_uv.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: base -# moonvision/python-base -# -# Dependencies include a small set of build-time -# dependencies to ease compiling libraries. -channels: - - conda-forge -dependencies: - # switch all blas requirements to use mkl - # implementation, see: https://conda-forge.org/docs/maintainer/knowledge_base.html#blas - # it doesn't constrain the libblas version (1st *), but the build string to end in 'mkl' (2nd *) - - libblas=*=*mkl