Reqs in pyproject

Author: benjeffery

.circleci/config.yml

@@ -13,10 +13,12 @@ commands:
             # Install meson to the system packages so we can run it as root
             pip install --upgrade pip wheel
             sudo pip install meson
-            # Needed for some build steps in requirements.txt, no blas as expensive to
+            # Install uv for dependency management
+            pip install --user uv
+            # Install dependencies from pyproject.toml, no blas as expensive to
             # build on 32 bit
             pip install --user numpy --config-settings=setup-args="-Dallow-noblas=true"
-            pip install --user -r python/requirements/CI-complete/requirements.txt --config-settings=setup-args="-Dallow-noblas=true"
+            cd python && uv pip install --system -r pyproject.toml --extra test --config-settings=setup-args="-Dallow-noblas=true"
             # Remove tskit installed by msprime
             pip uninstall tskit -y
             echo 'export PATH=/home/circleci/.local/bin:$PATH' >> $BASH_ENV

.github/workflows/docs.yml

@@ -8,8 +8,7 @@ on:
       - '*'
 jobs:
   Docs:
-    uses: tskit-dev/.github/.github/workflows/docs-build-template.yml@main
+    uses: tskit-dev/.github/.github/workflows/docs-build-template.yml@v1
     with:
-      requirements-path: python/requirements/CI-docs/requirements.txt
       additional-setup: sudo apt-get install -y doxygen
       make-command: make -C python

.github/workflows/tests.yml

@@ -53,23 +53,6 @@ jobs:
         run: |
           brew install libxml2 libxslt
 
-      - name: Cache conda and dependencies
-        id: cache
-        uses: actions/[email protected]
-        with:
-          path: ${{ env.CONDA }}/envs
-          key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.python}}-conda-v1-${{ hashFiles('python/requirements/CI-tests-conda/requirements.txt') }}-${{ hashFiles('python/requirements/CI-tests-pip/requirements.txt') }}
-
-      - name: Install Conda
-        uses: conda-incubator/[email protected]
-        if: steps.cache.outputs.cache-hit != 'true'
-        with:
-          activate-environment: anaconda-client-env
-          python-version: ${{ matrix.python }}
-          channels: conda-forge
-          channel-priority: strict
-          auto-update-conda: true
-
       - name: Fix windows symlinks
         working-directory: python
         if: matrix.os == 'windows-latest'
@@ -80,47 +63,36 @@ jobs:
           cp -r --dereference ../c/tskit lib/.
           cp ../c/tskit.h lib/.
 
-      - name: Fix windows .profile
-        if: steps.cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-        run: |
-          cp ~/.bash_profile ~/.profile
-
-      - name: Install conda deps
-        if: steps.cache.outputs.cache-hit != 'true'
-        shell: bash -l {0} #We need a login shell to get conda
-        run: |
-          conda install --yes --file=python/requirements/CI-tests-conda/requirements.txt
-
-      - name: Install pip deps
-        if: steps.cache.outputs.cache-hit != 'true'
-        shell: bash -l {0}
+      - name: Install uv and set the python version
+        uses: astral-sh/setup-uv@v6
+        with:
+          python-version: ${{ matrix.python }}
+          version: "0.8.15"
+      
+      - name: Install dependencies
+        working-directory: python
         run: |
-          pip install -r python/requirements/CI-tests-pip/requirements.txt
-          # Remove tskit installed by conda
-          pip uninstall -y tskit
+          uv venv
+          uv pip install -r pyproject.toml --extra test
+          # Remove tskit installed as a dep of msprime
+          uv pip uninstall tskit
 
       - name: Build module
         working-directory: python
         run: |
-          source ~/.profile
-          conda activate anaconda-client-env
-          python setup.py build_ext --inplace
+          uv run --no-sync python setup.py build_ext --inplace
 
       - name: Minidom test
         working-directory: python
         # Importing either IPython or pytest causes import of xml.dom.minidom
         # So to actually test that tskit imports it, we need a minimal test
         run: |
-          source ~/.profile
-          conda activate anaconda-client-env
-          python -c "import tskit;tskit.Tree.generate_star(5).tree_sequence.draw_svg(path='test.svg')"
+          uv run --no-sync python -c "import tskit;tskit.Tree.generate_star(5).tree_sequence.draw_svg(path='test.svg')"
 
       - name: Run JIT code coverage
         working-directory: python
         run: |
-          source ~/.profile
-          conda activate anaconda-client-env
-          NUMBA_DISABLE_JIT=1 python -m pytest -x --cov=tskit --cov-report=xml --cov-branch -n2 tests/test_jit.py
+          NUMBA_DISABLE_JIT=1 uv run --no-sync python -m pytest -x --cov=tskit --cov-report=xml --cov-branch -n2 tests/test_jit.py
 
       - name: Upload coverage to Codecov
         uses: codecov/[email protected]
@@ -134,12 +106,10 @@ jobs:
       - name: Run tests
         working-directory: python
         run: |
-          source ~/.profile
-          conda activate anaconda-client-env
           if [[ "${{ matrix.os }}" == "windows-latest" ]]; then
-            python -m pytest -x --skip-slow --cov=tskit --cov-report=xml --cov-branch -n2 --durations=20 tests
+            uv run --no-sync python -m pytest -x --skip-slow --cov=tskit --cov-report=xml --cov-branch -n2 --durations=20 tests
           else
-            python -m pytest -x --cov=tskit --cov-report=xml --cov-branch -n2 --durations=20 tests
+            uv run --no-sync python -m pytest -x --cov=tskit --cov-report=xml --cov-branch -n2 --durations=20 tests
           fi
 
       - name: Upload coverage to Codecov
@@ -173,21 +143,28 @@ jobs:
         with:
           python-version: '3.12'
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.8.15"
+
       - name: Install dependencies
         working-directory: python
         run: |
-          pip install -r requirements/CI-complete/requirements.txt
-          pip install "numpy<2"
+          uv venv
+          uv pip install -r pyproject.toml --extra test
+          uv pip uninstall tskit
+          uv pip install "numpy<2"
 
       - name: Build module
         working-directory: python
         run: |
-          python setup.py build_ext --inplace
+          uv run --no-sync python setup.py build_ext --inplace
 
       - name: Run tests with numpy 1.x
         working-directory: python
         run: |
-          python -m pytest -x --cov=tskit --cov-report=xml --cov-branch -n2 tests/test_lowlevel.py tests/test_highlevel.py
+          uv run --no-sync python -m pytest -x --cov=tskit --cov-report=xml --cov-branch -n2 tests/test_lowlevel.py tests/test_highlevel.py
 
       - name: Upload coverage to Codecov
         uses: codecov/[email protected]

.pyup.yml

@@ -78,32 +78,29 @@ with:
 $ sudo apt install python3-dev python-is-python3 build-essential doxygen
 ```
 
-Python packages required for development are listed in `python/requirements/development.txt`.
-These can be installed using `pip`. We recommend using an isolated environment as below:
+The packages needed for development are specified as optional dependencies
+in the ``python/pyproject.toml`` file. Install these using:
 
 ```bash
 $ python3 -m venv env
 $ source env/bin/activate
-$ python3 -m pip install -r python/requirements/development.txt
+$ cd python
+$ python3 -m pip install -e ".[dev]"
 ```
 
 A few extra dependencies are required if you wish to work on the
 {ref}`C library <sec_development_c_requirements>`.
 
-For OSX and Windows users we recommending using
+For OSX and Windows users we recommend using
 [conda](https://docs.conda.io/projects/conda/en/latest/),
 and isolating development in a dedicated environment as follows:
 
 ```bash
-$ conda env create -f python/requirements/development.yml
+$ conda env create -f python/development.yml
 $ conda activate tskit-dev
 ```
 
-On macOS, conda builds are generally done using `clang` packages that are kept up to date:
-
-```bash
-$ conda install clang_osx-64  clangxx_osx-64
-```
+The environment includes macOS-specific clang packages when needed.
 
 In order to make sure that these compilers work correctly (*e.g.*, so that they can find
 other dependencies installed via `conda`), you need to compile `tskit` with this command
@@ -442,9 +439,8 @@ Test code is contained in the `tests` directory. Tests are also roughly split
 by function, so that tests for the `drawing` module are in the
 `tests/test_drawing.py` file. This is not a one-to-one mapping, though.
 
-The `requirements` directory contains descriptions of the requirements
-needed for development and on various
-{ref}`sec_development_continuous_integration` providers.
+Development dependencies are specified in the `pyproject.toml` file
+and can be installed using pip or conda (via `development.yml`).
 
 (sec_development_python_style)=
 
@@ -651,7 +647,7 @@ $ sudo apt install libcunit1-dev ninja-build meson clang-format-6.0
     if so, you can install it instead with `pip` by running
     `pip3 install clang-format==6.0.1 && ln -s clang-format $(which clang-format)-6.0`.
 
-Conda users can install the basic requirements from `python/requirements/development.txt`.
+Conda users can install the development environment using `python/development.yml`.
 
 Unfortunately clang-format is not available on conda, but it is not essential.
 

docs/development.md

@@ -0,0 +1,22 @@
+name: tskit-dev
+channels:
+  - conda-forge
+dependencies:
+  - python>=3.9
+  - pip
+  - doxygen
+  - numpy
+  - pandas
+  - scipy
+  - pytest
+  - pytest-cov
+  - pytest-xdist
+  - coverage
+  - flake8
+  - mypy
+  - pre-commit
+  - sphinx>=4.4
+  - jupyter-book
+  - networkx
+  - matplotlib
+  - jsonschema

python/development.yml

@@ -62,6 +62,73 @@ packages = ["tskit"]
 [tool.setuptools.dynamic]
 version = {attr = "tskit._version.tskit_version"}
 
+[project.optional-dependencies]
+test = [
+    "biopython==1.85",
+    "coverage==7.7.0", 
+    "dendropy==5.0.1",
+    "kastore==0.3.3",
+    "lshmm==0.0.8",
+    "msgpack==1.1.0",
+    "msprime==1.3.4",
+    "networkx==3.2.1",
+    "numba==0.61.2; python_version>='3.10'",
+    "numba==0.60.0; python_version=='3.9'",
+    "portion==2.6.0",
+    "pytest==8.3.5",
+    "pytest-cov==6.0.0",
+    "pytest-xdist==3.6.1",
+    "tszip==0.2.5",
+    "xmlunittest==1.0.1",
+    "svgwrite==1.4.3",
+    "newick==1.10.0",
+    "zarr<3",
+]
+
+docs = [
+    "jupyter-book==1.0.4.post1",
+    "breathe==4.35.0",
+    "sphinx-autodoc-typehints==2.3.0",
+    "sphinx-issues==5.0.0",
+    "sphinx-argparse==0.5.2",
+    "msprime==1.3.3",
+    "numba==0.61.2",
+    "sphinx-book-theme",
+    "pandas==2.2.3",
+]
+
+dev = [
+    "biopython>=1.70",
+    "coverage",
+    "dendropy",
+    "flake8",
+    "kastore",
+    "lshmm",
+    "msgpack",
+    "msprime",
+    "mypy",
+    "networkx",
+    "numba",
+    "portion",
+    "pre-commit",
+    "pytest",
+    "pytest-cov",
+    "pytest-xdist", 
+    "setuptools_scm",
+    "svgwrite",
+    "tszip",
+    "xmlunittest",
+    "newick",
+    "zarr<3",
+    "jupyter-book",
+    "breathe",
+    "sphinx-autodoc-typehints",
+    "sphinx-issues",
+    "sphinx-argparse",
+    "sphinx-book-theme",
+    "pandas",
+]
+
 [tool.pytest.ini_options]
 addopts = "-n 4"
 testpaths = ["tests"]