Update Docker config to install dependencies in separate service. (#590)

Also drop support for Python 3.8 and switch to Python 3.13 in development.

Author: david-yz-liu

Changelog.md

@@ -7,6 +7,9 @@ All notable changes to this project will be documented here.
 - Improve display of Python package installation errors when creating environment (#585)
 - Update "setting up test environment" message with http response of status code 503 (#589)
 - Change rlimit resource settings to apply each worker individually (#587) 
+- Drop support for Python 3.8 (#590)
+- Use Python 3.13 in development (#590)
+- Update Docker configuration to install dependencies in a separate service (#590)
 - Improve error reporting with handled assertion errors (#591)
 - Add custom pytest markers to Python tester to record MarkUs metadata (#592)
 - Stop the autotester from running tests if there are errors in test settings (#593) 

client/.dockerfiles/Dockerfile

@@ -5,13 +5,7 @@ FROM ubuntu:$UBUNTU_VERSION
 RUN apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get -y install software-properties-common && \
     DEBIAN_FRONTEND=noninteractive add-apt-repository -y ppa:deadsnakes/ppa && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y python3.11 python3.11-venv
-
-COPY ./requirements.txt /requirements.txt
-
-RUN python3.11 -m venv /markus_venv && \
-    /markus_venv/bin/pip install wheel && \
-    /markus_venv/bin/pip install -r /requirements.txt
+    DEBIAN_FRONTEND=noninteractive apt-get install -y python3.13 python3.13-venv
 
 WORKDIR /app
 

client/.dockerfiles/entrypoint-dev-deps-updater.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+[ -f /markus_venv/bin/python3.13 ] || python3.13 -m venv /markus_venv
+
+/markus_venv/bin/pip install -q --upgrade pip
+/markus_venv/bin/pip install -q wheel
+printf "[MarkUs] Running pip install -q -r requirements.txt..."
+if /markus_venv/bin/pip install -q -r requirements.txt; then
+  printf " \e[32m✔\e[0m \n"
+fi
+
+# Execute the provided command
+exec "$@"

client/requirements.txt

@@ -1,10 +1,6 @@
-flask==2.2.5;python_version<"3.8"
-flask==3.1.0;python_version>="3.8"
-python-dotenv==0.21.1;python_version<"3.8"
-python-dotenv==1.0.1;python_version>="3.8"
+flask==3.1.0
+python-dotenv==1.0.1
 rq==2.1.0
 redis==5.2.1
-jsonschema==4.17.3;python_version<"3.8"
-jsonschema==4.23.0;python_version>="3.8"
-Werkzeug==2.2.3;python_version<"3.8"
-Werkzeug==3.1.3;python_version>="3.8"
+jsonschema==4.23.0
+Werkzeug==3.1.3

compose.yaml

@@ -1,15 +1,16 @@
 services:
-  server:
+  server: &server
     build:
       context: ./server
       dockerfile: ./.dockerfiles/Dockerfile
       args:
         UBUNTU_VERSION: '22.04'
         LOGIN_USER: 'docker'
         WORKSPACE: '/home/docker/.autotesting'
-    image: markus-autotest-server-dev:1.2.0
+    image: markus-autotest-server-dev:1.3.0
     volumes:
       - ./server:/app:cached
+      - venv_server:/home/docker/markus_venv
       - workspace:/home/docker/.autotesting:rw
     environment:
       - REDIS_URL=redis://redis:6379/
@@ -22,16 +23,17 @@ services:
       - postgres
       - redis
 
-  client:
+  client: &client
     build:
       context: ./client
       dockerfile: ./.dockerfiles/Dockerfile
       args:
         UBUNTU_VERSION: '22.04'
-    image: markus-autotest-client-dev:1.2.0
+    image: markus-autotest-client-dev:1.3.0
     container_name: 'autotest-client'
     volumes:
       - ./client:/app:cached
+      - venv_client:/markus_venv
     environment:
       - REDIS_URL=redis://redis:6379/
       - FLASK_HOST=0.0.0.0
@@ -44,6 +46,20 @@ services:
       - default
       - markus_dev
 
+  server-deps-updater:
+    <<: *server
+    entrypoint: "/app/.dockerfiles/entrypoint-dev-deps-updater.sh"
+    profiles: ["manual"]  # Prevent service from starting automatically
+    depends_on: []
+
+  client-deps-updater:
+    <<: *client
+    entrypoint: ".dockerfiles/entrypoint-dev-deps-updater.sh"
+    profiles: ["manual"]  # Prevent service from starting automatically
+    ports: []
+    depends_on: []
+    networks: []
+
   postgres:
     image: postgres:14
     volumes:
@@ -73,6 +89,8 @@ services:
 volumes:
   postgres_autotest:
   redis_autotest:
+  venv_client:
+  venv_server:
   workspace:
 
 networks:

server/.dockerfiles/Dockerfile

@@ -43,15 +43,12 @@ RUN chmod a+x /home/${LOGIN_USER}
 
 COPY . /app
 
-RUN python3.11 -m venv /markus_venv && \
-    /markus_venv/bin/pip install wheel && \
-    /markus_venv/bin/pip install -r /app/requirements.txt && \
-    find /app/autotest_server/testers -name requirements.system -exec {} \; && \
-    rm -rf /app/*
+RUN find /app/autotest_server/testers -name requirements.system -exec {} \;
 
 RUN echo "TZ=$( cat /etc/timezone )" >> /etc/R/Renviron.site
 
 RUN mkdir -p ${WORKSPACE} && chown ${LOGIN_USER} ${WORKSPACE}
+RUN mkdir -p /home/${LOGIN_USER}/markus_venv && chown ${LOGIN_USER} /home/${LOGIN_USER}/markus_venv
 
 WORKDIR /home/${LOGIN_USER}
 

server/.dockerfiles/cmd-dev.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
-/markus_venv/bin/python /app/install.py
-/markus_venv/bin/python /app/start_stop.py start -n
+markus_venv/bin/python /app/install.py
+markus_venv/bin/python /app/start_stop.py start -n

server/.dockerfiles/entrypoint-dev-deps-updater.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+[ -f markus_venv/bin/python3.13 ] || python3.13 -m venv markus_venv
+
+markus_venv/bin/pip install -q --upgrade pip
+markus_venv/bin/pip install -q wheel
+printf "[MarkUs] Running pip install -q -r /app/requirements.txt..."
+if markus_venv/bin/pip install -q -r /app/requirements.txt; then
+  printf " \e[32m✔\e[0m \n"
+fi
+
+# Execute the provided command
+exec "$@"

server/requirements.txt

@@ -1,11 +1,8 @@
 rq==2.1.0
 click==8.1.8
 redis==5.2.1
-pyyaml==6.0.1;python_version<"3.8"
-pyyaml==6.0.2;python_version>="3.8"
-jsonschema==4.17.3;python_version<"3.8"
-jsonschema==4.23.0;python_version>="3.8"
-requests==2.31.0;python_version<"3.8"
-requests==2.32.3;python_version>="3.8"
+pyyaml==6.0.2
+jsonschema==4.23.0
+requests==2.32.3
 psycopg2-binary==2.9.10
 supervisor==4.2.5