Merge pull request #72 from olivierphi/uv-test

[chore] Switching from Poetry to uv, and from Black to Ruff - big Astral commitment! 😅

Author: olivierphi

.dockerignore

@@ -1,20 +1,23 @@
 /.venv
 
+/bin
+
 /node_modules
 
 /frontend-out
 /staticfiles
 
 /db.sqlite3
 
-/src/apps/*/static/css
-/src/apps/*/static/js
-/src/apps/*/static/chess
+/src/apps/*/static/*/css
+/src/apps/*/static/*/js
 
 /.git
 /.idea
 /.docker
 
+*.local
+
 **/.*_cache
 **/__pycache__
 **/*.pyc

.github/workflows/test-suite.yml

@@ -15,36 +15,31 @@ env:
 
 jobs:
   tests:
-    name: "Python ${{ matrix.python-version }}"
+    name: "Code quality and tests checks"
     runs-on: "ubuntu-latest"
 
-    strategy:
-      matrix:
-        python-version: ["3.11"]
-
     steps:
       - uses: "actions/checkout@v4"
-      - name: "Install poetry"
-        env:
-          POETRY_VERSION: "1.8.3"
-        run: pipx install poetry==${POETRY_VERSION}        
-      - uses: "actions/setup-python@v5"
-        with:
-          python-version: "${{ matrix.python-version }}"
-          cache: 'poetry' 
-      - name: "Install dependencies via Poetry"
-        run: poetry install
-      - name: "Run linting checks: Black"
-        run: poetry run black src/
-      - name: "Run linting checks: Ruff"
-        run: poetry run ruff check --quiet src/
+      - name: Set up uv
+        # Install a specific uv version using the installer
+        run: "curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh"
+        env: 
+          UV_VERSION: "0.4.4"
+      - name: Set up Python
+        run: uv python install
+      - name: "Install dependencies via uv"
+        run: uv sync --all-extras
+      - name: "Run linting checks: Ruff checker"
+        run: uv run ruff format --check --quiet src/
+      - name: "Run linting checks: Ruff linter"
+        run: uv run ruff check --quiet src/
       - name: "Run linting checks: Mypy"
-        run: poetry run mypy src/
+        run: uv run mypy src/
       - name: "Check that Django DB migrations are up to date"
-        run: poetry run python src/manage.py makemigrations | grep "No changes detected"
+        run: uv run python manage.py makemigrations | grep "No changes detected"
       - name: "Run tests"
         # TODO: progressively  increase minimum coverage to something closer to 80%
-        run: poetry run pytest --cov=src --cov-report xml:coverage.xml
+        run: uv run pytest --cov=src --cov-report xml:coverage.xml
         # --cov-fail-under=60 --> we'll actually do that with the "Report coverage" step
       - name: "Report coverage"
         # @link https://github.com/orgoro/coverage

.gitignore

@@ -9,6 +9,12 @@
 
 /.venv/
 
+# uv-related stuff:
+/bin/uv
+/bin/uvx
+/.uv.env
+/.uv.env.fish
+
 /.docker/*
 !/.docker/.gitkeep
 

.pre-commit-config.yaml

@@ -1,23 +1,22 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
-  - repo: https://github.com/psf/black
-    rev: 24.1.1
-    hooks:
-      - id: black
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.8.0
+    rev: v1.11.2
     hooks:
       - id: mypy
         additional_dependencies: [types-requests==2.31.0.2]
         exclude: "^scripts/load_testing/.*\\.py$"
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.1.14
+    rev: v0.6.3
     hooks:
+      # Run the formatter.
+      - id: ruff-format
+      # Run the linter.
       - id: ruff
         args: ["--fix"] 
         exclude: "^src/project/settings/.*\\.py$"
   - repo: https://github.com/abravalheri/validate-pyproject
-    rev: v0.16
+    rev: v0.19
     hooks:
       - id: validate-pyproject

.python-version

@@ -0,0 +1 @@
+3.11

Dockerfile

@@ -53,34 +53,63 @@ RUN make frontend/img frontend/js/compile frontend/css/compile \
 
 #########################################################################
 # Backend stuff
+# Large chunks copy-pasted from https://hynek.me/articles/docker-uv/ :-)
 
 FROM python:3.11-slim-bookworm AS backend_build
 
-ENV POETRY_VERSION=1.8.3
-
-ENV DEBIAN_FRONTEND=noninteractive
-ENV PYTHONDONTWRITEBYTECODE=0 PYTHONUNBUFFERED=1
-
-RUN apt-get update && apt-get install -y \
-    python3-pip \
-    python3-venv \
-    python3-dev \
-    python3-setuptools \
-    python3-wheel \
-    libpq-dev \
-    && apt-get clean && rm -rf /var/lib/apt/lists/*
-
-RUN pip install --upgrade pip
-RUN pip install poetry==${POETRY_VERSION}
+# The following does not work in Podman unless you build in Docker
+# compatibility mode: <https://github.com/containers/podman/issues/8477>
+# You can manually prepend every RUN script with `set -ex` too.
+SHELL ["sh", "-exc"]
+
+RUN <<EOT
+apt-get update -qy
+apt-get install -qyy \
+    -o APT::Install-Recommends=false \
+    -o APT::Install-Suggests=false \
+    build-essential \
+    ca-certificates \
+    curl
+EOT
+
+# Install uv.
+# https://docs.astral.sh/uv/guides/integration/docker/
+COPY --from=ghcr.io/astral-sh/uv:0.4.4 /uv /usr/local/bin/uv
 
 RUN mkdir -p /app
 WORKDIR /app
 
-RUN python -m venv --symlinks .venv
-
-COPY pyproject.toml poetry.lock ./
-RUN poetry install --only=main --no-root --no-interaction --no-ansi
-
+# - Silence uv complaining about not being able to use hard links,
+# - tell uv to byte-compile packages for faster application startups,
+# - prevent uv from accidentally downloading isolated Python builds,
+# - pick a Python,
+# - and finally declare `/app/.venv` as the target for `uv sync`.
+ENV UV_LINK_MODE=copy \
+    UV_COMPILE_BYTECODE=1 \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_PYTHON=python3.11 \
+    UV_PROJECT_ENVIRONMENT=/app/.venv
+    
+# Prepare a virtual environment.
+# This is cached until the Python version changes above.
+RUN --mount=type=cache,target=/root/.cache \
+    uv venv $UV_PROJECT_ENVIRONMENT
+
+# Synchronize DEPENDENCIES without the application itself.
+# This layer is cached until uv.lock or pyproject.toml change.
+# Since there's no point in shipping lock files, we move them
+# into a directory that is NOT copied into the runtime image.
+COPY pyproject.toml /_lock/
+COPY uv.lock /_lock/
+RUN --mount=type=cache,target=/root/.cache <<EOT
+cd /_lock
+uv sync \
+    --frozen \
+    --no-dev \
+    --no-install-project
+EOT
+
+    
 FROM python:3.11-slim-bookworm AS assets_download
 
 # By having a separate build stage for downloading assets, we can cache them
@@ -100,21 +129,29 @@ RUN python scripts/download_assets.py
 
 FROM python:3.11-slim-bookworm AS backend_run
 
-ENV DEBIAN_FRONTEND=noninteractive
-ENV PYTHONDONTWRITEBYTECODE=0 PYTHONUNBUFFERED=1
+SHELL ["sh", "-exc"]
 
-RUN apt-get update && apt-get install -y \
-    libpq5 \
-    && apt-get clean && rm -rf /var/lib/apt/lists/*
+# Add the application virtualenv to search path.
+ENV PATH=/app/.venv/bin:$PATH
 
-RUN mkdir -p /app
-WORKDIR /app
+# Allow our "src/" folder to be seen as a package root:
+ENV PYTHONPATH=/app/src
 
-RUN addgroup -gid 1001 webapp
-RUN useradd --gid 1001 --uid 1001 webapp
-RUN chown -R 1001:1001 /app
+# Let's make sure that our Django app sees "/app" as its base dir, rather than
+# a resolved symlink in the venv. This is important for the collectstatic command.
+ENV DJANGO_BASE_DIR=/app
+
+# Don't run our app as root.
+RUN <<EOT
+groupadd --gid 1001 app
+useradd --uid 1001 -g app -N app -d /app
+EOT
+
+USER app
+WORKDIR /app
 
 COPY --chown=1001:1001 scripts scripts
+COPY --chown=1001:1001 Makefile pyproject.toml manage.py LICENSE ./
 COPY --chown=1001:1001 src src
 
 COPY --chown=1001:1001 --from=frontend_build /app/src/apps/webui/static src/apps/webui/static
@@ -127,24 +164,24 @@ COPY --chown=1001:1001 --from=assets_download /app/src/apps/chess/static/chess/j
 COPY --chown=1001:1001 --from=assets_download /app/src/apps/chess/static/chess/units src/apps/chess/static/chess/units
 COPY --chown=1001:1001 --from=assets_download /app/src/apps/chess/static/chess/symbols src/apps/chess/static/chess/symbols
 
-COPY --chown=1001:1001 Makefile pyproject.toml LICENSE ./
-
-ENV PATH="/app/.venv/bin:${PATH}"
 RUN python -V
+RUN python -Im site
+# Smoke test:
+RUN DJANGO_SETTINGS_MODULE=project.settings.docker_build \
+    .venv/bin/python manage.py shell -c "from apps.daily_challenge.models import DailyChallenge"
 
-USER 1001:1001
-
-# Install the project's packages (defined in pyproject.toml) in editable mode:
-RUN pip install -e .
-
+# Collect static files.
 RUN DJANGO_SETTINGS_MODULE=project.settings.docker_build \
-    .venv/bin/python src/manage.py collectstatic --noinput
+    .venv/bin/python manage.py collectstatic --noinput
 
+# Ok, let's get ready to run that server!
 EXPOSE 8080
 
 ENV DJANGO_SETTINGS_MODULE=project.settings.production
 
-ENV GUNICORN_CMD_ARGS="--bind :8080 --workers 2 --max-requests 120 --max-requests-jitter 20 --timeout 8"
+ENV GUNICORN_CMD_ARGS="--bind 0.0.0.0:8080 --workers 2 --max-requests 120 --max-requests-jitter 20 --timeout 8"
 
 RUN chmod +x scripts/start_server.sh
-CMD ["scripts/start_server.sh"]
+# See <https://hynek.me/articles/docker-signals/>.
+STOPSIGNAL SIGINT
+ENTRYPOINT ["scripts/start_server.sh"]

Makefile

@@ -10,9 +10,10 @@ help:
 	@grep -P '^[.a-zA-Z/_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
 .PHONY: install
-install: .venv ./node_modules ## Install the Python and frontend dependencies
+install: bin/uv .venv ./node_modules ## Install the Python and frontend dependencies
+	bin/uv sync --all-extras
 	${PYTHON_BINS}/pre-commit install
-	${PYTHON_BINS}/poetry install
+	${SUB_MAKE} .venv/bin/black
 
 .PHONY: dev
 dev: .env.local db.sqlite3
@@ -60,19 +61,19 @@ test: ## Launch the pytest tests suite
 		${PYTHON_BINS}/pytest ${pytest_opts}
 
 .PHONY: code-quality/all
-code-quality/all: code-quality/black code-quality/ruff code-quality/mypy  ## Run all our code quality tools
+code-quality/all: code-quality/ruff_check code-quality/ruff_lint code-quality/mypy  ## Run all our code quality tools
 
-.PHONY: code-quality/black
-code-quality/black: black_opts ?=
-code-quality/black: ## Automated 'a la Prettier' code formatting
-# @link https://black.readthedocs.io/en/stable/
-	@${PYTHON_BINS}/black ${black_opts} src/
+.PHONY: code-quality/ruff_check
+code-quality/ruff_check: ruff_opts ?=
+code-quality/ruff_check: ## Automated 'a la Prettier' code formatting
+# @link https://docs.astral.sh/ruff/formatter/
+	@${PYTHON_BINS}/ruff format ${ruff_opts} src/
 
-.PHONY: code-quality/ruff
-code-quality/ruff: ruff_opts ?= --fix
-code-quality/ruff: ## Fast linting
-# @link https://mypy.readthedocs.io/en/stable/
-	@${PYTHON_BINS}/ruff src/ ${ruff_opts}
+.PHONY: code-quality/ruff_lint
+code-quality/ruff_lint: ruff_opts ?= --fix
+code-quality/ruff_lint: ## Fast linting
+# @link https://docs.astral.sh/ruff/linter/
+	@${PYTHON_BINS}/ruff check src/ ${ruff_opts}
 
 .PHONY: code-quality/mypy
 code-quality/mypy: mypy_opts ?=
@@ -142,14 +143,22 @@ frontend/img/copy_assets:
 
 # Here starts the "misc util targets" stuff
 
-.venv: poetry_version ?= 1.8.3
-.venv: ## Initialises the Python virtual environment in a ".venv" folder
-	python -m venv .venv
-	${PYTHON_BINS}/pip install -U pip poetry==${poetry_version}
+bin/uv: uv_version ?= 0.4.4
+bin/uv: # Install `uv` and `uvx` locally in the "bin/" folder
+	curl -LsSf "https://astral.sh/uv/${uv_version}/install.sh" | \
+		CARGO_DIST_FORCE_INSTALL_DIR="$$(pwd)" INSTALLER_NO_MODIFY_PATH=1 sh
+	@echo "We'll use 'bin/uv' to manage Python dependencies." 
+
+.venv: ## Initialises the Python virtual environment in a ".venv" folder, via uv
+	bin/uv venv
 
 .env.local:
 	cp .env.dist .env.local
 
+.venv/bin/black: .venv ## A simple and stupid shim to use the IDE's Black integration with Ruff
+	@echo '#!/usr/bin/env sh\n$$(dirname "$$0")/ruff format $$@' > ${PYTHON_BINS}/black
+	@chmod +x ${PYTHON_BINS}/black
+
 db.sqlite3: dotenv_file ?= .env.local
 db.sqlite3: ## Initialises the SQLite database
 	touch db.sqlite3
@@ -164,7 +173,7 @@ django/manage: cmd ?= --help
 django/manage: .venv .env.local ## Run a Django management command
 	@DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} ${env_vars} \
 		${PYTHON_BINS}/dotenv -f '${dotenv_file}' run -- \
-		${PYTHON} src/manage.py ${cmd}
+		${PYTHON} manage.py ${cmd}
 
 ./.venv/bin/django: .venv install
 
@@ -247,13 +256,13 @@ docker/local/shell:
 		-u ${user_id} \
 		${docker_env} ${docker_args} \
 		-e DJANGO_SETTINGS_MODULE=project.settings.production \
-		${DOCKER_IMG_NAME}:${DOCKER_TAG} \
-		${cmd}
+		--entrypoint ${cmd} \
+		${DOCKER_IMG_NAME}:${DOCKER_TAG}
 
 .PHONY: docker/local/migrate
 docker/local/migrate:
 	${SUB_MAKE} docker/local/shell \
-		cmd='/app/.venv/bin/python src/manage.py migrate'
+		cmd='/app/.venv/bin/python manage.py migrate'
 
 # Here starts Fly.io-related stuff
 .PHONY: fly.io/deploy

README.md

@@ -39,10 +39,11 @@ The game lives at [zakuchess.com](https://zakuchess.com).
 
 ### Running locally
 
-Make sure you have Python 3.11 installed, as well as Node.js v18.
-
-We recommend [pyenv](https://github.com/pyenv/pyenv-installer#readme) and [nvm](https://github.com/nvm-sh/nvm#readme) to handle specific versions of Python and Node.js,
-but you can of course use whatever you want :-)
+ - Python: Entirely managed by [uv](https://docs.astral.sh/uv/). The `make install` command will
+    make sure that the right version of `uv` is installed in the "bin/" folder - which is not ignored by git.
+ - Node.js: Make sure you have Node.js v18 installed.  
+     We recommend [nvm](https://github.com/nvm-sh/nvm#readme) to handle specific versions of Python and Node.js,
+     but you can of course use whatever you want :-)
 
 ```bash
 $ python -V