Remove pypy in favor of cython

ldeluigi · ldeluigi · commit 63445e94f963 · 2026-01-13T18:09:53.000+01:00
diff --git a/.dockerignore b/.dockerignore
@@ -12,6 +12,14 @@
 **/*_data.json
 **/.*_cache
 **/*.log
+**/*$py.class
+**/*.c
+**/*.so
+**/*.py[cod]
+**/*.pdb
+**/*.whl
+**/*.egg
+**/*.egg-info
 hidden/
 docs/
 demo/
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -106,8 +106,6 @@ jobs:
           - '3.13'
           - '3.12'
           - '3.11'
-          - 'pypy3.11'
-          - 'pypy3.10'
         os:
           - ubuntu-latest
           - windows-latest
@@ -141,7 +139,7 @@ jobs:
         working-directory: backend
 
       - name: Unit Test 🧪
-        run: python -Wd manage.py test --no-input ${{ !startsWith(matrix.python-version, 'pypy') && '--parallel auto' || '' }} --pythonpath ../common
+        run: python -Wd manage.py test --no-input --parallel auto --pythonpath ../common
         working-directory: backend
 
 
diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,7 @@ __pycache__/
 *$py.class
 
 # C extensions
+*.c
 *.so
 
 # Distribution / packaging
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -167,28 +167,6 @@
                 "order": 4
             },
         },
-        {
-            "name": "Generate Variants using PyPy",
-            "type": "debugpy",
-            "python": "pypy",
-            "request": "launch",
-            "cwd": "${workspaceFolder}/backend",
-            "program": "manage.py",
-            "args": [
-                "generate_variants"
-            ],
-            "env": {
-                "PYTHONPATH": "${workspaceFolder}${/}common",
-            },
-            "preLaunchTask": "Prelaunch",
-            "django": true,
-            "justMyCode": true,
-            "presentation": {
-                "hidden": false,
-                "group": "Spellbook",
-                "order": 4
-            },
-        },
         {
             "name": "Profile Generate Variants",
             "type": "debugpy",
@@ -210,28 +188,6 @@
                 "order": 4
             },
         },
-        {
-            "name": "Profile Generate Variants using PyPy",
-            "type": "debugpy",
-            "python": "pypy",
-            "request": "launch",
-            "cwd": "${workspaceFolder}/backend",
-            "module": "cProfile",
-            "args": [
-                "-o", "${workspaceFolder}/backend/temp/profiles/generate_variants_pypy.prof", "manage.py", "generate_variants"
-            ],
-            "env": {
-                "PYTHONPATH": "${workspaceFolder}${/}common",
-            },
-            "preLaunchTask": "Prelaunch",
-            "django": true,
-            "justMyCode": true,
-            "presentation": {
-                "hidden": false,
-                "group": "Spellbook",
-                "order": 5
-            },
-        },
         {
             "name": "Export variants",
             "type": "debugpy",
diff --git a/backend/Dockerfile b/backend/Dockerfile
@@ -1,7 +1,13 @@
+##########
+# PYTHON #
+##########
+FROM python:3.12-alpine AS python
+
+
 ###########
 # BUILDER #
 ###########
-FROM python:3.12-alpine AS builder
+FROM python AS builder
 
 # set work directory
 WORKDIR /usr/src/app
@@ -20,22 +26,10 @@ COPY backend/requirements.txt .
 RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/python/wheels -r requirements.txt
 
 
-#################
-# PYPY BUILDER  #
-#################
-FROM builder AS pypy-builder
-
-RUN echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
-    apk --update add pypy3@testing
-
-RUN pypy3 -m ensurepip && pypy3 -m pip install --upgrade pip
-
-RUN pypy3 -m pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/pypy/wheels $(grep -ivE "#|cryptography|cffi" requirements.txt)
-
 #########
 # BASE  #
 #########
-FROM python:3.12-alpine AS base
+FROM python AS base
 
 # create the app user
 RUN addgroup -S app && adduser -S app -G app
@@ -74,32 +68,37 @@ ENTRYPOINT ["/bin/sh", "entrypoint.prod.sh"]
 RUN echo -e 'gunicorn backend.wsgi:application --bind 0.0.0.0:8000\n' > entrypoint.prod.sh
 
 
-#########
-# PYPY  #
-#########
-FROM base AS pypy
+##########
+# CYTHON #
+##########
+FROM builder AS cython
 
-# Switch to user root in order to install pypy
-USER root
+RUN pip install --upgrade cython setuptools
 
-# install dependencies
-RUN echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
-    apk --update add pypy3@testing && \
-    ln -s $(which pypy3) $(dirname $(which pypy3))/pypy && \
-    pypy -m ensurepip && pypy -m pip install --no-cache-dir --upgrade pip
 
-COPY --from=pypy-builder /usr/src/app/pypy/wheels /pypy/wheels
+##################
+# CYTHON-BUILDER #
+##################
 
-RUN pypy -m pip install --no-cache-dir /pypy/wheels/* --no-deps
-RUN pypy -m pip install --no-cache-dir psycopg
+FROM cython AS cython-builder
+
+COPY backend/ common/ /usr/src/app/
+WORKDIR /usr/src/app/
+RUN cythonize -i 'spellbook/variants/*.py'
+
+
+###############
+# BASE-CYTHON #
+###############
+FROM base AS base-cython
+
+COPY --from=cython-builder --parents /usr/src/app/./**/*.so $APP_HOME
 
-# Switch back to the app user
-USER app
 
 #########
 # DEMO  #
 #########
-FROM pypy AS demo
+FROM base-cython AS demo
 
 RUN echo -e 'set -e\n\
 python manage.py collectstatic --no-input --clear\n\
@@ -113,7 +112,7 @@ gunicorn backend.wsgi:application --bind 0.0.0.0:8000\n' > entrypoint.prod.sh
 ##############
 # PRODUCTION #
 ##############
-FROM pypy AS production
+FROM base-cython AS production
 
 ADD backend/nginx/production.conf /etc/nginx/nginx.conf
 
diff --git a/backend/backend/settings.py b/backend/backend/settings.py
@@ -12,7 +12,6 @@
 import os
 from pathlib import Path
 from datetime import timedelta
-from common.pypy_utils import PYPY_AVAILABLE as check_pypy
 from constants import SORTED_COLORS
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
@@ -30,8 +29,6 @@
 
 STATIC_BULK_FOLDER = Path('./temp/bulk')
 
-PYPY_AVAILABLE = check_pypy
-
 VERSION = os.getenv('VERSION', 'dev')
 
 ALLOWED_HOSTS = ['*']
diff --git a/backend/common/pypy_utils.py b/backend/common/pypy_utils.py
diff --git a/backend/requirements.txt b/backend/requirements.txt
@@ -121,6 +121,8 @@ sqlparse==0.5.0
     # via django
 typing-extensions==4.15.0
     # via referencing
+tzdata==2025.3
+    # via django
 uritemplate==4.2.0
     # via drf-spectacular
 urllib3==2.6.3
diff --git a/backend/spellbook/models/utils.py b/backend/spellbook/models/utils.py
@@ -1,6 +1,5 @@
 import re
 import unicodedata
-from platform import python_implementation
 from typing import Callable, Generator, Iterable
 from collections import defaultdict
 from ..regexs import MANA_SYMBOL, ORACLE_SYMBOL, ORACLE_SYMBOL_EXTENDED
@@ -302,23 +301,11 @@ def remove_random_from_order_by(order_by: Iterable[str | F | OrderBy]) -> Genera
 def __default_batch_size() -> int:
     '''Returns an appropriate batch size for the current database connection.'''
     otherwise = 100
-    match python_implementation():
-        case 'PyPy':
-            match connection.vendor:
-                case 'sqlite':
-                    return 1000
-                case 'postgresql':
-                    return 45
-                case _:
-                    return otherwise
-        case 'CPython':
-            match connection.vendor:
-                case 'sqlite':
-                    return 1000
-                case 'postgresql':
-                    return 275
-                case _:
-                    return otherwise
+    match connection.vendor:
+        case 'sqlite':
+            return 1000
+        case 'postgresql':
+            return 275
         case _:
             return otherwise
 
diff --git a/backend/spellbook/utils.py b/backend/spellbook/utils.py
@@ -13,8 +13,7 @@
 
 def launch_command_async(command: str, args: list[str] = []):
     manage_py_path = pathlib.Path(__file__).parent.parent / 'manage.py'
-    avoid_pypy = command in ['runserver', 'shell', 'export_variants']
-    interpreter = 'pypy' if not avoid_pypy and settings.PYPY_AVAILABLE else 'python'
+    interpreter = 'python'
     args = [interpreter, manage_py_path.resolve(), command] + args
     if sys.platform == "win32":
         subprocess.Popen(
diff --git a/backend/spellbook/variants/minimal_set_of_multisets.py b/backend/spellbook/variants/minimal_set_of_multisets.py
@@ -1,4 +1,4 @@
-from typing import Iterable, TypeVar, Generic
+from typing import Iterable, Self, TypeVar, Generic
 from .multiset import FrozenMultiset
 
 _T = TypeVar('_T')
@@ -90,7 +90,7 @@ def copy(self):
         "Returns a shallow copy of the collection."
         return self.__copy__()
 
-    def __or__(self, other: 'MinimalSetOfMultisets[_T]'):  # TODO: replace with Self from typing with pypy 3.11
+    def __or__(self, other: Self):
         result = self.copy()
         result.extend(other)
         return result
diff --git a/backend/spellbook/variants/variant_set.py b/backend/spellbook/variants/variant_set.py
@@ -1,4 +1,4 @@
-from typing import Iterable, Callable
+from typing import Iterable, Callable, Self
 from itertools import product, chain
 from functools import reduce
 from dataclasses import dataclass
@@ -77,11 +77,11 @@ def __str__(self) -> str:
     def __len__(self) -> int:
         return len(self.sets)
 
-    def __or__(self, other: 'VariantSet'):  # TODO: replace with Self from typing with pypy 3.11
+    def __or__(self, other: Self):
         assert self.parameters == other.parameters, "Cannot union VariantSets with different parameters"
         return self.__class__(parameters=self.parameters, _internal=self.sets | other.sets)
 
-    def __and__(self, other: 'VariantSet'):  # TODO: replace with Self from typing with pypy 3.11
+    def __and__(self, other: Self):
         assert self.parameters == other.parameters, "Cannot intersect VariantSets with different parameters"
         result = MinimalSetOfMultisets[int]()
         for left_entry, right_entry in product(self.entries(), other.entries()):
@@ -91,7 +91,7 @@ def __and__(self, other: 'VariantSet'):  # TODO: replace with Self from typing w
             result.add(entry)
         return self.__class__(parameters=self.parameters, _internal=result)
 
-    def __add__(self, other: 'VariantSet'):  # TODO: replace with Self from typing with pypy 3.11
+    def __add__(self, other: Self):
         assert self.parameters == other.parameters, "Cannot sum VariantSets with different parameters"
         result = MinimalSetOfMultisets[int]()
         for left_key, right_key in product(self.entries(), other.entries()):
@@ -105,25 +105,25 @@ def variants(self) -> list[tuple[FrozenMultiset[cardid], FrozenMultiset[template
         return [self.entry_to_ingredients(e) for e in self.entries()]
 
     @classmethod
-    def or_sets(cls, sets: list['VariantSet'], parameters: VariantSetParameters | None = None):  # TODO: replace with Self from typing with pypy 3.11
+    def or_sets(cls, sets: list[Self], parameters: VariantSetParameters | None = None):
         return cls.aggregate_sets(sets, strategy=lambda x, y: x | y, parameters=parameters)
 
     @classmethod
-    def and_sets(cls, sets: list['VariantSet'], parameters: VariantSetParameters | None = None):  # TODO: replace with Self from typing with pypy 3.11
+    def and_sets(cls, sets: list[Self], parameters: VariantSetParameters | None = None):
         return cls.aggregate_sets(sets, strategy=lambda x, y: x & y, parameters=parameters)
 
     @classmethod
-    def sum_sets(cls, sets: list['VariantSet'], parameters: VariantSetParameters | None = None):  # TODO: replace with Self from typing with pypy 3.11
+    def sum_sets(cls, sets: list[Self], parameters: VariantSetParameters | None = None):
         return cls.aggregate_sets(sets, strategy=lambda x, y: x + y, parameters=parameters)
 
     @classmethod
-    def aggregate_sets(cls, sets: list['VariantSet'], strategy: Callable[['VariantSet', 'VariantSet'], 'VariantSet'], parameters: VariantSetParameters | None = None):  # TODO: replace with Self from typing with pypy 3.11
-        match len(sets):
-            case 0: return cls(parameters=parameters)
-            case _: return reduce(strategy, sets)
+    def aggregate_sets(cls, sets: list[Self], strategy: Callable[[Self, Self], Self], parameters: VariantSetParameters | None = None):
+        if len(sets) == 0:
+            return cls(parameters=parameters)
+        return reduce(strategy, sets)
 
     @classmethod
-    def product_sets(cls, sets: list['VariantSet'], parameters: VariantSetParameters | None = None):  # TODO: replace with Self from typing with pypy 3.11
+    def product_sets(cls, sets: list[Self], parameters: VariantSetParameters | None = None):
         parameters = parameters if parameters is not None else VariantSetParameters()
         if parameters.allow_multiple_copies:
             return cls.sum_sets(sets, parameters=parameters)
diff --git a/docs/index.md b/docs/index.md
@@ -28,5 +28,4 @@ You need:
 - `flake8` for linting the code, which is mandatory for contributing otherwise the CI will fail
 - [Optional] VS Code with the Python extension, for development and debugging exploiting the `launch.json` configuration
     - `pytest` to run the unit tests in VS Code
-- [Optional] PyPy 3.10 or later for running long tasks faster
-  - Remember to install the dependencies using `pypy -m pip install -r requirements.txt` in the backend folder
+- [Optional] Cython for running long tasks faster (especially the variant generation code)
diff --git a/docs/minimal-set-of-multisets.md b/docs/minimal-set-of-multisets.md
@@ -68,4 +68,4 @@ These are the papers/links to refer for the implementation of an optimized MSM (
 
 ## Considerations
 
-PyPy works way better with simple Python code than with over-engineered data structures, so the currently fastest implementation of the MSM is the naive one, which uses a Python set to store the multisets and checks for supersets/subsets using simple loops.
+Cython works way better with simple Python code than with over-engineered data structures, so the currently fastest implementation of the MSM is the naive one, which uses a Python set to store the multisets and checks for supersets/subsets using simple loops.
