Docker compose setup improvements + fix GitHub actions (#317)

* chore: Small improvements in the docker-compose setup

- remove the network used by all containers (dc creates it by default)
- set default DATABASE_TYPE to sqlite => simpler dc up commands
- change restart policy to unless-stopped => containers don't restart
  after system restart (which was annoying in local development)

* fix: Fix the initial SQL data import for local docker-compose setup

* fix: added psycog2 package to init container and set grader network

* fix: Remove postgres-specific syntax from the initial data file

`E'\n'` is invalid syntax in sqlite. It was used in
`replace(<very long logs as a string>, '\n', E'\n')` and
`replace(<very long logs as a string>, '\012', E'\n')`.
I just replaced these values (`\n` and `\012`) with newlines.

* cleaup: Remove psycopg from Dockerfile (not needed, after all)

* fix: Fix database initialisation in docker compose for sqlite and postgres

* fix: update docker compose docs

* fix: Fix a comment

* chore: Use uv to install python packages in containers in Docker Compose setup

* chore: Make the various init bash scripts for containers more consistent

* chore: Also use uv in jupyterhub Dockerfile

* cleanup: Remove a redundant `expose` from docker compose file

* fix: Remove submission data from the initial db import

We don't include the submission files and repos in the initial setup,
and creating the database objects without them doesn't make much sense,
because one cannot do anything with such submissions in grader service.

* fix: Simplify python deps installation in Dockerfile-Service

* fix: Only load one assignment as initial db data in docker compose setup

* fix: Set the initial data assignment deadline to a future date

* fix: Small adjustment in the init assignment data

* chore: Add initial 'source' repo for docker compose setup

* chore: Add initial 'release' repo for docker compose setup

* chore: Make initial repos available in the docker compose setup

* fix: Fix celery worker container command

* fix: Add `test` dependency group to pyproject.toml, upgrade dev and test deps.

* fix: Replace deprecated `set-output` with $GITHUB_OUTPUT

* chore: Delete unused .gitlab-ci.yml file

* fix: Tiny improvements in docker compose setup (better caching, get rid of a warning)

* fix: Fix github action check for Dockerfile changes

Previously, the check would only output directory names, so the check
for the "Dockerfile" name would never match. I added a separate step that
checks specifically if the Dockerfile has changed.

---------

Co-authored-by: florian.jaeger <[email protected]>

Author: natkam

.github/workflows/action_main.yml

@@ -30,18 +30,33 @@ jobs:
         id: changeDirsStep
         uses: tj-actions/changed-files@v46
         with:
-          dir_names: true
+          dir_names: true  # this only outputs directory names
           files: |
             **/*
             *
+
+      - name: Check if Dockerfile changed
+        id: changeDockerfileStep
+        uses: tj-actions/changed-files@v46
+        with:
+          files: Dockerfile
+
+
       - id: outputStep
-        run: echo "::set-output name=changeDirs::${{ steps.changeDirsStep.outputs.all_changed_files }}"
+        run: |
+          {
+            echo "changeDirs=${{ steps.changeDirsStep.outputs.all_changed_files }}";
+            echo "changeDockerfile=${{ steps.changeDockerfileStep.outputs.any_changed }}"
+          } >> "$GITHUB_OUTPUT"
 
   # ON CHANGED FILES
   build_grader-service:
     uses: ./.github/workflows/build.yml
     needs: init
-    if: contains(needs.init.outputs.changeDirs, 'grader_service') || contains(needs.init.outputs.changeDirs, '.github/workflows') || contains(needs.init.outputs.changeDirs, 'Dockerfile')
+    if: |
+      contains(needs.init.outputs.changeDirs, 'grader_service') ||
+      contains(needs.init.outputs.changeDirs, '.github/workflows') ||
+      needs.init.outputs.changeDockerfile == 'true'
 
   dockerize_grader-service:
     needs: build_grader-service

.gitlab-ci.yml

@@ -17,7 +17,7 @@ the containers together with their services.
 
 Run the following command:
 ```bash
-DATABASE_TYPE=sqlite docker compose up --build -d
+docker compose up --build -d
 ```
 
 This command builds the images and starts the containers in detached mode. It sets SQLite as its database that is integrated in `Grader Service`.
@@ -43,5 +43,3 @@ To stop and remove the containers, run:
 ```bash
 docker compose -f docker-compose.yml -f docker-compose-postgres.yml down -v
 ```
-
-

docs/source/installation/docker.md

@@ -1,20 +1,24 @@
 FROM postgres
 
-# Install git
+# Install python
 RUN apt-get update && \
-    apt-get install -y python3 python3-venv && \
+    apt-get install -y python3 && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+# Install uv
+COPY --from=ghcr.io/astral-sh/uv:0.9.3 /uv /uvx /bin/
+
 WORKDIR /app
 
-# Copy necessary files and folders
+# Install python dependencies, only copying necessary files
 COPY ./pyproject.toml MANIFEST.in ./
-COPY ./grader_service ./grader_service
-# Create a virtual environment
-RUN python3 -m venv /venv
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --no-install-project --link-mode=copy
 
-# Activate venv and install the package
-RUN /venv/bin/pip install .
+# Copy the project files and install the package itself
+COPY ./grader_service ./grader_service
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --link-mode=copy .
 
-CMD ["sh", "-c", "/app/init-db.sh"]
+CMD ["sh", "-c", "/app/init-db-postgres.sh"]

examples/docker_compose/Dockerfile-DBInit

@@ -6,12 +6,14 @@ RUN apt-get update && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+# Install uv
+COPY --from=ghcr.io/astral-sh/uv:0.9.3 /uv /uvx /bin/
+
 WORKDIR /app
 
 ENV JUPYTERHUB_CRYPT_KEY=e3f92a0d5e37446c7e894a2ef6c6ec3bcb4aa3c38d2a442eb567adf780597aa0
 
-RUN pip install grader_labextension
-
-RUN python3 -m pip install dockerspawner oauthenticator
+RUN  --mount=type=cache,target=/root/.cache/uv \
+    uv pip install grader_labextension dockerspawner oauthenticator
 
-CMD ["jupyterhub", "-f", "/app/jupyterhub_config.py"]
+CMD ["jupyterhub", "-f", "/app/jupyterhub_config.py"]

examples/docker_compose/Dockerfile-Hub

@@ -1,19 +1,26 @@
 FROM python:3.13-slim
 
-# Install git
+# Install git and sqlite
 RUN apt-get update && \
-    apt-get install -y git && \
+    apt-get install -y git sqlite3 && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+# Install uv
+COPY --from=ghcr.io/astral-sh/uv:0.9.3 /uv /uvx /bin/
+ENV UV_SYSTEM_PYTHON=true
+
 WORKDIR /app
 
-# Copy necessary files and folders
+# Install python dependencies, only copying necessary files
 COPY ./pyproject.toml MANIFEST.in ./
-COPY ./grader_service ./grader_service
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --no-install-project
 
-RUN pip install .
-RUN pip install numpy ipykernel
+# Copy the project files and install only the package itself
+COPY ./grader_service ./grader_service
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --link-mode=copy -e .
 
 # Set default branch to main and default user
 RUN git config --global init.defaultBranch main && \

examples/docker_compose/Dockerfile-Service

@@ -16,7 +16,7 @@ the containers together with their services.
 
 Run the following command:
 ```bash
-DATABASE_TYPE=sqlite docker compose up --build -d
+docker compose up --build -d
 ```
 
 This command builds the images and starts the containers in detached mode. It sets SQLite as its database that is integrated in `Grader Service`.
@@ -42,5 +42,3 @@ To stop and remove the containers, run:
 ```bash
 docker compose -f docker-compose.yml -f docker-compose-postgres.yml down -v
 ```
-
-