MP-1314 Add builds for Python 3.13

.github/workflows/build.yml

@@ -23,13 +23,13 @@ jobs:
       with:
         imagename: moonvision/${{ env.IMAGE_PRE }}python-base
         path: docker/python-base
-        dockerfile: docker/python-base/Dockerfile
+        dockerfile: docker/python-base/DockerfileMicromamba
         baseimage: mambaorg/micromamba:1
     - uses: ./.github/create-docker
       with:
         imagename: moonvision/${{ env.IMAGE_PRE }}python-base
         path: docker/python-base
-        dockerfile: docker/python-base/Dockerfile
+        dockerfile: docker/python-base/DockerfileMicromamba
         baseimage: mambaorg/micromamba:1
         additionaltag: 3.11
         additional_buildargs: env_file=environment_3.11.yml
@@ -41,17 +41,25 @@ jobs:
         with:
           imagename: moonvision/${{ env.IMAGE_PRE }}python-base
           path: docker/python-base
-          dockerfile: docker/python-base/DockerfileUV
+          dockerfile: docker/python-base/Dockerfile
           additionaltag: uv
           additional_buildargs: python_version=3.8
       - uses: ./.github/create-docker
         with:
           imagename: moonvision/${{ env.IMAGE_PRE }}python-base
           path: docker/python-base
-          dockerfile: docker/python-base/DockerfileUV
+          dockerfile: docker/python-base/Dockerfile
           additionaltag: uv-3.11
           additional_buildargs: |-
             python_version=3.11
+      - uses: ./.github/create-docker
+        with:
+          imagename: moonvision/${{ env.IMAGE_PRE }}python-base
+          path: docker/python-base
+          dockerfile: docker/python-base/Dockerfile
+          additionaltag: uv-3.13
+          additional_buildargs: |-
+            python_version=3.13
   build-ffmpeg:
     runs-on: ubuntu-latest
     steps:
@@ -193,7 +201,7 @@ jobs:
         imagename: moonvision/${{ env.IMAGE_PRE }}moonbox
         additionaltag: mini
         path: docker/moonbox
-        dockerfile: docker/moonbox/Dockerfile
+        dockerfile: docker/moonbox/DockerfileMicromamba
         baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:latest
         additional_buildargs: |-
           ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-4.2.1
@@ -211,7 +219,7 @@ jobs:
           imagename: moonvision/${{ env.IMAGE_PRE }}moonbox
           additionaltag: mini-3.11
           path: docker/moonbox
-          dockerfile: docker/moonbox/Dockerfile
+          dockerfile: docker/moonbox/DockerfileMicromamba
           baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:3.11-latest
           additional_buildargs: |-
             ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-4.2.1
@@ -229,7 +237,7 @@ jobs:
         imagename: moonvision/${{ env.IMAGE_PRE }}moonbox
         additionaltag: cuda
         path: docker/moonbox
-        dockerfile: docker/moonbox/Dockerfile
+        dockerfile: docker/moonbox/DockerfileMicromamba
         baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:latest
         additional_buildargs: |-
           with_cuda=true
@@ -248,7 +256,7 @@ jobs:
         imagename: moonvision/${{ env.IMAGE_PRE }}moonbox
         additionaltag: cuda-3.11
         path: docker/moonbox
-        dockerfile: docker/moonbox/Dockerfile
+        dockerfile: docker/moonbox/DockerfileMicromamba
         baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:3.11-latest
         additional_buildargs: |-
           with_cuda=true
@@ -266,7 +274,7 @@ jobs:
         imagename: moonvision/${{ env.IMAGE_PRE }}moonbox
         additionaltag: mini-uv
         path: docker/moonbox
-        dockerfile: docker/moonbox/DockerfileUV
+        dockerfile: docker/moonbox/Dockerfile
         baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:uv-latest
         additional_buildargs: |-
           ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-4.2.1
@@ -282,10 +290,26 @@ jobs:
           imagename: moonvision/${{ env.IMAGE_PRE }}moonbox
           additionaltag: mini-uv-3.11
           path: docker/moonbox
-          dockerfile: docker/moonbox/DockerfileUV
+          dockerfile: docker/moonbox/Dockerfile
           baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:uv-3.11-latest
           additional_buildargs: |-
             ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-4.2.1
+  docker-moonbox-mini-uv-313:
+    needs:
+      - docker-python-base-uv
+      - build-ffmpeg
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/create-docker
+        with:
+          imagename: moonvision/${{ env.IMAGE_PRE }}moonbox
+          additionaltag: mini-uv-3.13
+          path: docker/moonbox
+          dockerfile: docker/moonbox/Dockerfile
+          baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:uv-3.13-latest
+          additional_buildargs: |-
+            ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-4.2.1
   docker-moonbox-cuda-uv:
     needs:
     - docker-python-base-uv
@@ -298,7 +322,7 @@ jobs:
         imagename: moonvision/${{ env.IMAGE_PRE }}moonbox
         additionaltag: cuda-uv
         path: docker/moonbox
-        dockerfile: docker/moonbox/DockerfileUV
+        dockerfile: docker/moonbox/Dockerfile
         baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:uv-latest
         additional_buildargs: |-
           with_cuda=true
@@ -315,8 +339,25 @@ jobs:
         imagename: moonvision/${{ env.IMAGE_PRE }}moonbox
         additionaltag: cuda-uv-3.11
         path: docker/moonbox
-        dockerfile: docker/moonbox/DockerfileUV
+        dockerfile: docker/moonbox/Dockerfile
         baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:uv-3.11-latest
         additional_buildargs: |-
           with_cuda=true
           ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-cuda-4.2.1
+  docker-moonbox-cuda-uv-313:
+    needs:
+    - docker-python-base-uv
+    - build-ffmpeg-cuda
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: ./.github/create-docker
+      with:
+        imagename: moonvision/${{ env.IMAGE_PRE }}moonbox
+        additionaltag: cuda-uv-3.13
+        path: docker/moonbox
+        dockerfile: docker/moonbox/Dockerfile
+        baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:uv-3.13-latest
+        additional_buildargs: |-
+          with_cuda=true
+          ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-cuda-4.2.1

docker/moonbox/Dockerfile

@@ -1,20 +1,22 @@
-ARG baseimage='moonvision/python-base'
+ARG baseimage='moonvision/python-base:uv-latest'
 ARG ffmpeg_from_docker='moonvision/custom-builds:ffmpeg-4.2.1'
-ARG pytorch_from_docker='moonvision/custom-builds:pytorch-1.11.0_torchvision-0.12.0'
 
 FROM $ffmpeg_from_docker as ffmpeg_builder
-FROM $pytorch_from_docker as pytorch_builder
 
 FROM ${baseimage} as installer
 
 ARG with_cuda='false'
 
+ARG DEBIAN_FRONTEND='noninteractive'
+ # fixme: Why should gcc be installed both here and in python-base? And the same for checkinstall.
+RUN apt-get update --yes --no-install-recommends \
+ && apt-get install --yes gcc g++ checkinstall cmake --no-install-recommends
+
 COPY --from=ffmpeg_builder /packages /ffmpeg-packages
-COPY --from=pytorch_builder /packages /pytorch-packages
 COPY . /bd_build
 
 ARG MAMBA_DOCKERFILE_ACTIVATE=1
-RUN with_cuda=$with_cuda bash /bd_build/install_deps.sh
+RUN with_cuda=$with_cuda bash /bd_build/install_deps_uv.sh
 
 # squash /packages
 FROM ${baseimage}

docker/moonbox/DockerfileUV → docker/moonbox/DockerfileMicromamba

@@ -1,22 +1,20 @@
-ARG baseimage='moonvision/python-base:uv-latest'
+ARG baseimage='moonvision/python-base'
 ARG ffmpeg_from_docker='moonvision/custom-builds:ffmpeg-4.2.1'
+ARG pytorch_from_docker='moonvision/custom-builds:pytorch-1.11.0_torchvision-0.12.0'
 
 FROM $ffmpeg_from_docker as ffmpeg_builder
+FROM $pytorch_from_docker as pytorch_builder
 
 FROM ${baseimage} as installer
 
 ARG with_cuda='false'
 
-ARG DEBIAN_FRONTEND='noninteractive'
- # fixme: Why should gcc be installed both here and in python-base? And the same for checkinstall.
-RUN apt-get update --yes --no-install-recommends \
- && apt-get install --yes gcc g++ checkinstall cmake --no-install-recommends
-
 COPY --from=ffmpeg_builder /packages /ffmpeg-packages
+COPY --from=pytorch_builder /packages /pytorch-packages
 COPY . /bd_build
 
 ARG MAMBA_DOCKERFILE_ACTIVATE=1
-RUN with_cuda=$with_cuda bash /bd_build/install_deps_uv.sh
+RUN with_cuda=$with_cuda bash /bd_build/install_deps.sh
 
 # squash /packages
 FROM ${baseimage}

docker/python-base/Dockerfile

@@ -1,9 +1,18 @@
-ARG baseimage='mambaorg/micromamba:1.0.0'
+ARG python_version
+# This stage is copied from mambaorg/micromamba Dockerfile and contains everything except the actual micromamba stuff
+FROM ghcr.io/astral-sh/uv:python${python_version}-bookworm-slim AS preparation_stage
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
 
-# Build runit-docker
-FROM ${baseimage} AS build_runit_docker
+ARG CERT_SOURCE='/etc/ssl/certs/ca-certificates.crt'
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+ENV ENV_NAME="base"
+
+# Default command for "docker run"
+CMD ["/bin/bash"]
 
-USER root
+# Build runit-docker
+FROM preparation_stage AS build_runit_docker
 
 RUN apt-get update \
  && apt-get install -y curl make gcc libc-dev checkinstall --no-install-recommends \
@@ -17,24 +26,9 @@ RUN cd /tmp \
  && mkdir /packages \
  && mv *.deb /packages
 
-FROM ${baseimage}
-
-# Install python
-COPY --chown=$MAMBA_USER:$MAMBA_USER . /home/$MAMBA_USER/bd_build
-ARG env_file='environment.yml'
-RUN micromamba install -y -f /home/$MAMBA_USER/bd_build/$env_file && \
-    micromamba clean -yaf
+FROM preparation_stage
 
-# Manifest mamba python for exec environments
-ENV CPATH "$MAMBA_ROOT_PREFIX/include"
-ENV PATH "$MAMBA_ROOT_PREFIX/bin:$PATH"
-
-USER root
-# You may seal production images by running:
-#USER $MAMBA_USER
-
-# adduser template of $MAMBA_USER
-RUN echo "SKEL=/home/$MAMBA_USER" >> /etc/default/useradd
+SHELL ["/bin/bash", "-c"]
 
 # Install other tools
 ARG DEBIAN_FRONTEND=noninteractive
@@ -48,6 +42,13 @@ RUN apt-get update -y \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
+# Configure uv
+# Without this you will have to pass `--no-sync` for each `uv run` call, because it will try to install dev requirements otherwise
+ENV UV_NO_SYNC=true
+# Next two are needed, because otherwise uv cannot find certificates
+ENV UV_NATIVE_TLS=true \
+    SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
+
 # Install runit-docker
 COPY --from=build_runit_docker /packages /packages
 RUN dpkg -i /packages/*.deb && rm -rf /packages

docker/python-base/DockerfileMicromamba

@@ -0,0 +1,54 @@
+ARG baseimage='mambaorg/micromamba:1.0.0'
+
+# Build runit-docker
+FROM ${baseimage} AS build_runit_docker
+
+USER root
+
+RUN apt-get update \
+ && apt-get install -y curl make gcc libc-dev checkinstall --no-install-recommends \
+ && apt-get clean
+
+RUN cd /tmp \
+ && curl -o runit-docker-1.2.tar.gz -L https://github.com/kosma/runit-docker/archive/refs/tags/1.2.tar.gz \
+ && tar xf runit-docker-1.2.tar.gz \
+ && cd runit-docker-1.2 \
+ && checkinstall --install=no \
+ && mkdir /packages \
+ && mv *.deb /packages
+
+FROM ${baseimage}
+
+# Install python
+COPY --chown=$MAMBA_USER:$MAMBA_USER . /home/$MAMBA_USER/bd_build
+ARG env_file='environment.yml'
+RUN micromamba install -y -f /home/$MAMBA_USER/bd_build/$env_file && \
+    micromamba clean -yaf
+
+# Manifest mamba python for exec environments
+ENV CPATH "$MAMBA_ROOT_PREFIX/include"
+ENV PATH "$MAMBA_ROOT_PREFIX/bin:$PATH"
+
+USER root
+# You may seal production images by running:
+#USER $MAMBA_USER
+
+# adduser template of $MAMBA_USER
+RUN echo "SKEL=/home/$MAMBA_USER" >> /etc/default/useradd
+
+# Install other tools
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -y \
+ && apt-get install -y --no-install-recommends \
+    tzdata \
+    libglib2.0-0 \
+    wget bzip2 curl git procps \
+    runit \
+  && apt-get purge --autoremove --yes python python3 \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+# Install runit-docker
+COPY --from=build_runit_docker /packages /packages
+RUN dpkg -i /packages/*.deb && rm -rf /packages
+CMD ["/usr/sbin/runit-docker"]