1110 Migrate Python packages to use pyproject.toml for builds (#1409)

- Replace setuptools `setup.py` with PEP 621 `pyproject.toml` across all Python subpackages.
- Move python tests outside of the subpackage directories to avoid import errors.
- Add a reusable lint helper for packages that used a custom pylint command.
- Update CI build/test actions.

Co-authored-by: patricklnz <patrick.lenz01@icloud.com>
Co-authored-by: reneSchm <49305466+reneSchm@users.noreply.github.com>

Author: 3 people

.github/actions/build-py/action.yml

@@ -19,9 +19,11 @@ runs:
       shell: bash
       run: |
         cd pycode/memilio-${{ inputs.package }}/
-        /opt/python/cp38-cp38/bin/python -m pip install scikit-build
+        /opt/python/cp38-cp38/bin/python -m pip install --upgrade pip setuptools wheel
+        /opt/python/cp38-cp38/bin/python -m pip install scikit-build scikit-build-core
         /opt/python/cp38-cp38/bin/python -m build --no-isolation --wheel
-        /opt/python/cp311-cp311/bin/python -m pip install scikit-build
+        /opt/python/cp311-cp311/bin/python -m pip install --upgrade pip setuptools wheel
+        /opt/python/cp311-cp311/bin/python -m pip install scikit-build scikit-build-core
         /opt/python/cp311-cp311/bin/python -m build --no-isolation --wheel
         # Exclude memilio-generation, because its a pure python package, cmake is only used in the build process to retrieve data from cpp
         if [[ -f "CMakeLists.txt" ]] && [ "${{ inputs.package }}" != "generation" ]; then 

.github/actions/test-py/action.yml

@@ -50,19 +50,20 @@ runs:
       shell: bash
       run: |
         export PYTHON_VERSION_NO_DOT=$(echo "${{ inputs.version }}" | sed 's/\.//g')
-        for pkg in `ls pycode/wheelhouse/*cp$PYTHON_VERSION_NO_DOT*.whl`; do python -m pip install $pkg; done # packages that contain native extensions are version specific
-        for pkg in `ls pycode/wheelhouse/*py3*.whl`; do python -m pip install $pkg; done # pure python packages are not version specific
-        pip install -r pycode/memilio-${{ inputs.package }}/requirements-dev.txt
+        shopt -s nullglob
+        for pkg in pycode/wheelhouse/*cp$PYTHON_VERSION_NO_DOT*.whl; do python -m pip install "$pkg"; done # packages that contain native extensions are version specific
+        for pkg in pycode/wheelhouse/*py3*.whl; do python -m pip install "$pkg"; done # pure python packages are not version specific
+        python -m pip install --upgrade-strategy only-if-needed --prefer-binary --find-links pycode/wheelhouse "memilio-${{ inputs.package }}[dev]"
     - name: Run unit tests
       shell: bash
       run: |
-        cd pycode/memilio-${{inputs.package}}/memilio/${{inputs.package}}_test
+        cd pycode/memilio-${{inputs.package}}/tests
         if [[ "${{ inputs.coverage }}" == "ON" ]]; then
           python -W ignore::DeprecationWarning -m coverage run --include=**/site-packages/memilio/* -m unittest
           python -m coverage report
           python -m coverage xml -o coverage_python.xml
           python -m coverage html -d coverage_python
-          cp -r coverage_python ../../../../
+          cp -r coverage_python ../../../
         else
           python -m unittest
         fi

.github/actions/test-pylint/action.yml

@@ -13,23 +13,29 @@ runs:
         sudo apt-get -qq update
         sudo apt-get -qq -y install python3-pip gnupg
         python -m pip install --upgrade pip
+    # Pylint runs against the prebuilt wheels in pycode/wheelhouse/*cp311*.
+    # So, use current latest supported Python version.
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
     - name: Download Python Wheels
       uses: actions/download-artifact@v4
       with:
         name: python-wheels-${{ inputs.package }}
+        path: pycode/wheelhouse
     - name: Install Python Wheels
       shell: bash
       run: |
-        for pkg in `ls pycode/wheelhouse/*cp311*.whl`; do python -m pip install $pkg; done # packages that contain native extensions are version specific
-        for pkg in `ls pycode/wheelhouse/*py3*.whl`; do python -m pip install $pkg; done # pure python packages are not version specific
-        pip install -r pycode/memilio-${{ inputs.package }}/requirements-dev.txt
+        shopt -s nullglob
+        for pkg in pycode/wheelhouse/*cp311*.whl; do python -m pip install "$pkg"; done # packages that contain native extensions are version specific
+        for pkg in pycode/wheelhouse/*py3*.whl; do python -m pip install "$pkg"; done # pure python packages are not version specific
+        python -m pip install --upgrade-strategy only-if-needed --prefer-binary --find-links pycode/wheelhouse "memilio-${{ inputs.package }}[dev]"
     - name: Run pylint
       shell: bash
       run: |
-        cd pycode/memilio-${{ inputs.package }}
-        mkdir -p build_pylint
-        python setup.py pylint
-        pylint-json2html -f jsonextended -o build_pylint/pylint.html < build_pylint/pylint_extended.json
+        python pycode/run_pylint.py --package-dir memilio-${{ inputs.package }}
+        pylint-json2html -f jsonextended -o pycode/memilio-${{ inputs.package }}/build_pylint/pylint.html < pycode/memilio-${{ inputs.package }}/build_pylint/pylint_extended.json
     - name: Upload Pylint Report
       uses: actions/upload-artifact@v4
       with:

.readthedocs.yaml

@@ -11,7 +11,7 @@ build:
   jobs:
     install:
       - pip install --exists-action=w --no-cache-dir -r docs/requirements-build.txt
-      - pip install --exists-action=w --no-cache-dir -r docs/requirements.txt --no-build-isolation
+      - pip install --exists-action=w --no-cache-dir -r docs/requirements.txt
 
 sphinx:
   configuration: docs/source/conf.py

cpp/thirdparty/CMakeLists.txt

@@ -37,6 +37,7 @@ if(MEMILIO_USE_BUNDLED_SPDLOG)
 
     if(NOT spdlog_POPULATED)
         FetchContent_Populate(spdlog)
+
         add_subdirectory(${spdlog_SOURCE_DIR} ${spdlog_BINARY_DIR} EXCLUDE_FROM_ALL)
     endif()
 
@@ -198,6 +199,7 @@ else()
 endif()
 
 find_package(sbml ${MEMILIO_SBML_VERSION} CONFIG)
+
 if(sbml_FOUND)
     set(MEMILIO_HAS_SBML ON)
     message(STATUS "Found libSBML: ${libsbml_INCLUDE_DIRS}")

docs/source/python/m-epidata.rst

@@ -85,7 +85,7 @@ When you start creating a new script:
 
 When you add a new script
 
-- Add an executable to the setup.py in "pycode/memilio-epidata".
+- Add a console script entry to the ``pycode/memilio-epidata/pyproject.toml`` file.
 - Add it to the cli_dict in getDataIntoPandasDataFrame.py.
     - Add a meaningful key for the new script.
     - for the dict value add a list in the form [comment to print when script is started, list of used parser arguments (optional)].

docs/source/python/m-generation.rst

@@ -30,7 +30,7 @@ For a successful build, the development libraries for Python need to be installe
 .. warning::
     Generation currently requires specifically version ``18.1.1`` of `libclang.so`, since the function ``create_ast`` in `ast.py <https://github.com/SciCompMod/memilio/blob/main/pycode/memilio-generation/memilio/generation/ast.py>`_ generates the abstract syntax tree using `clang-18`. Different versions may lead to unsupported abstractions.
 
-    If you want to try a different version, set your `libclang` version under ``install_requires`` in the `setup.py <https://github.com/SciCompMod/memilio/blob/main/pycode/memilio-generation/setup.py>`_ and change the clang command in ``create_ast`` in `ast.py <https://github.com/SciCompMod/memilio/blob/main/pycode/memilio-generation/memilio/generation/ast.py>`_.
+    If you want to try a different version, set your `libclang` version under ``dependencies`` in the `pyproject.toml <https://github.com/SciCompMod/memilio/blob/main/pycode/memilio-generation/pyproject.toml>`_ and change the clang command in ``create_ast`` in `ast.py <https://github.com/SciCompMod/memilio/blob/main/pycode/memilio-generation/memilio/generation/ast.py>`_.
 
 Usage
 -----

docs/source/python/python_packages.rst

@@ -111,21 +111,21 @@ Please see the individual package documentation for more details on the function
 Installation
 ------------
 
-Each package provides a `setup.py` script that installs the package and its dependencies. 
+Each package provides a ``pyproject.toml`` that installs the package and its dependencies with pip.
 The dependencies of the individual packages are denoted in their documentation.
-The installation can be run with the following command (from the directory containing the `setup.py`)
+The installation can be run with the following command (from the directory containing the ``pyproject.toml`` file)
 
 .. code-block:: console 
     
-    pip install .
+    python -m pip install .
 
 This copies the package and the required dependencies to your site-packages.
 
 For development of code use this command instead
 
 .. code-block:: console 
     
-    pip install -e .[dev]
+    python -m pip install -e .[dev]
 
 This command allows you to work on the code without having to reinstall the package after a change. It also installs all additional dependencies required for development and maintenance.
 
@@ -157,7 +157,7 @@ To get the coverage report do in the package folder
 
 Coverage report for actual master:
 
-`Coverage Report <https://scicompmod.github.io/memilio/coverage/python/>`_
+`Coverage Report <https://scicompmod.github.io/memilio/coverage/python/>`__
 
 Inspection via pylint
 ---------------------
@@ -171,9 +171,12 @@ Run pylint with the commands in the package folder
 
 .. code-block:: console
 
-    python setup.py pylint
+    python ../run_pylint.py
     pylint-json2html -f jsonextended -o build_pylint/pylint.html < build_pylint/pylint_extended.json
 
+From the repository root you can also target a package explicitly, for example
+``python pycode/run_pylint.py --package-dir memilio-plot``.
+
 Pylint report for actual master:
 
-`Pylint Report <https://dlr-sc.github.io/memilio/pylint/>`_
+`Pylint Report <https://dlr-sc.github.io/memilio/pylint/>`__

pycode/memilio-epidata/README.rst

@@ -23,9 +23,9 @@ A more detailed description of the sources can be found in the `epidata subfolde
 Installation
 ------------
 
-Use the provided ``setup.py`` script to install the package and its dependencies.
+This project uses ``pyproject.toml`` to install the package and its dependencies.
 
-To install the package, use (from the directory that contains ``setup.py``)
+To install the package, run (from the directory that contains ``pyproject.toml``)
 
 .. code:: sh
 
@@ -84,11 +84,21 @@ The following packages are used by the tests:
 
 See Installation on how to install all these dependencies automatically.
 
-To run the tests make 
-
-.. code:: sh
+To run the tests, make sure the package is installed, and you are in the source directory, then run:
 
+```bash
+    cd tests
     python -m unittest
+```
+
+This works with both normal (`pip install .`) and editable (`pip install -e .`) installations.
+
+Alternatively, you can run the tests from outside the source directory:
+
+```bash
+cd /path/to/another/directory
+python -m unittest discover -s /path/to/memilio/pycode/memilio-epidata/tests
+```
 
 To get the coverage report do
 
@@ -116,7 +126,7 @@ Run pylint with the commands
 
 .. code:: sh
 
-    python setup.py pylint
+    python ../run_pylint.py
     pylint-json2html -f jsonextended -o build_pylint/pylint.html < build_pylint/pylint_extended.json
 
 Pylint report for actual master:
@@ -163,7 +173,7 @@ When you start creating a new script:
 
 When you add a new script
 
-- add a executable to the setup.py in "pycode/memilio-epidata"
+- add a script entry to ``pyproject.toml`` in "pycode/memilio-epidata"
 - add it to the cli_dict in getDataIntoPandasDataFrame.py
     - add a meaningfull key for the new script
     - as the value add a list in the form [comment to print when script is started, list of used parser arguments (optional)]