diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..887a2c18f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# SCM syntax highlighting & preventing 3-way merges +pixi.lock merge=binary linguist-language=YAML linguist-generated=true diff --git a/.github/workflows/test-spras.yml b/.github/workflows/test-spras.yml index db6d1b0fc..4c9ba3109 100644 --- a/.github/workflows/test-spras.yml +++ b/.github/workflows/test-spras.yml @@ -3,29 +3,26 @@ name: Test SPRAS on: [push, pull_request] jobs: - # Installs the conda environment but does not run tests because the tests require Linux Docker images - conda-only: - name: Test conda environment + # Installs the pixi environment but does not run tests because the tests require Linux Docker images + pixi-only: + name: Test pixi environment runs-on: ${{ matrix.os }} strategy: matrix: os: [macos-latest, windows-latest] steps: - name: Checkout repository - uses: actions/checkout@v2 - - name: Install conda environment - uses: conda-incubator/setup-miniconda@v2 + uses: actions/checkout@v4 + - name: Install pixi + uses: prefix-dev/setup-pixi@v0.9.0 with: - activate-environment: spras - environment-file: environment.yml - auto-activate-base: false - miniconda-version: 'latest' - - name: Log conda environment - # Log conda environment contents - shell: bash --login {0} - run: conda list + pixi-version: v0.53.0 + cache: false + locked: false + - name: Log pixi environment contents + run: pixi list - # Runs the test code and Snakemake workflow in the conda environment + # Runs the test code and Snakemake workflow in the pixi environment test: name: Run tests # The Docker images will be pulled in both the docker job and this test job @@ -48,21 +45,10 @@ jobs: sudo rm -rf /usr/share/dotnet /usr/local/lib/android sudo docker image prune --all --force sudo docker builder prune -a - - name: Install conda environment - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: spras - environment-file: environment.yml - auto-activate-base: false - miniconda-version: 'latest' - - name: Install spras in conda env - # Install spras in the environment using pip - shell: bash --login {0} - run: pip install . - - name: Log conda environment - # Log conda environment contents - shell: bash --login {0} - run: conda list + - name: Install pixi + uses: prefix-dev/setup-pixi@v0.9.0 + - name: Log pixi environment contents + run: pixi list - name: Install Apptainer # Formerly used Singularity instead of Apptainer (see https://github.com/eWaterCycle/setup-singularity/issues/6) uses: eWaterCycle/setup-apptainer@v2 @@ -70,10 +56,9 @@ jobs: # Choose version from https://github.com/apptainer/apptainer/releases apptainer-version: 1.3.6 - name: Run tests - shell: bash --login {0} # Verbose output and disable stdout and stderr capturing # [this pytest-split workflow was borrowed from Snakemake CI] - run: pytest -vs --splits 2 --group ${{ matrix.test_group }} + run: pixi run pytest -vs --splits 2 --group ${{ matrix.test_group }} workflow: name: Run workflow runs-on: ${{ matrix.os }} @@ -84,23 +69,19 @@ jobs: # TODO: is it worth it to make this setup into a composite action? - name: Checkout repository uses: actions/checkout@v4 - - name: Install conda environment - uses: conda-incubator/setup-miniconda@v2 + - name: Install pixi + uses: prefix-dev/setup-pixi@v0.9.0 with: - activate-environment: spras - environment-file: environment.yml - auto-activate-base: false - miniconda-version: 'latest' - # Install spras in the environment using pip - - name: Install spras in conda env - shell: bash --login {0} - run: pip install . + pixi-version: v0.53.0 + cache: false + locked: false + - name: Log pixi environment contents + run: pixi list - name: Run Snakemake workflow - shell: bash --login {0} # We enable high parallelization (cores 4) to test our way out of the experienced # race conditions from #268 and #279 # We also enforce strict DAG evaluation to catch DAG problems before they appear as user errors. (#359) - run: snakemake --cores 4 --configfile config/config.yaml --show-failed-logs --strict-dag-evaluation cyclic-graph --strict-dag-evaluation functions --strict-dag-evaluation periodic-wildcards + run: pixi run snakemake --cores 4 --configfile config/config.yaml --show-failed-logs --strict-dag-evaluation cyclic-graph --strict-dag-evaluation functions --strict-dag-evaluation periodic-wildcards # Run pre-commit checks on source files pre-commit: @@ -109,14 +90,12 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Install conda environment - uses: conda-incubator/setup-miniconda@v2 + - name: Install pixi + uses: prefix-dev/setup-pixi@v0.9.0 with: - activate-environment: spras - environment-file: environment.yml - auto-activate-base: false - miniconda-version: 'latest' + pixi-version: v0.53.0 + cache: false + locked: false - name: Run pre-commit - shell: bash --login {0} # https://github.com/pre-commit/action/blob/576ff52938d158a24ac7e009dfa94b1455e7df99/action.yml#L19 - run: pre-commit run --all-files --show-diff-on-failure --color=always + run: pixi run pre-commit run --all-files --show-diff-on-failure --color=always diff --git a/.gitignore b/.gitignore index 3629c49c6..d0aa78eb0 100644 --- a/.gitignore +++ b/.gitignore @@ -144,3 +144,12 @@ TempMat.mat # Singularity cache unpacked + +# pixi environments +.pixi +pixi.lock +*.egg-info + +# pixi environments +.pixi +*.egg-info diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5e92479d2..a3d851c1a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,6 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks +exclude: (^.pixi/|.snap) default_language_version: # Match this to the version specified in environment.yml python: python3.11 diff --git a/README.md b/README.md index 44895c76b..c6e1918e8 100644 --- a/README.md +++ b/README.md @@ -26,31 +26,20 @@ SPRAS requires First, download or clone this repository so that you have the Snakefile, example config file, and example data. -The easiest way to install Python and the required packages is with [Anaconda](https://www.anaconda.com/download/). -The Carpentries [Anaconda installation instructions](https://carpentries.github.io/workshop-template/#python) provide guides and videos on how to install Anaconda for your operating system. -After installing Anaconda, you can run the following commands from the root directory of the `spras` repository +The easiest way to install Python and the required packages is with [Pixi](https://pixi.sh/latest/installation/). +Optionally, after installing Pixi, you can run the following commands from the root directory of the `spras` repository ``` -conda env create -f environment.yml -conda activate spras +pixi shell ``` -to create a conda environment with the required packages and activate that environment. -If you have a different version of Python already, you can install the specified versions of the required packages in your preferred manner instead of using Anaconda. +to create a pixi environment that contains all of the packages you need to use SPRAS without having to prefix them with `pixi run`. +If you have a different version of Python already, you can install the specified versions of the required packages, as well as `go`, in your preferred manner instead of using Pixi. -While the `spras` conda environment comes bundled with all of Python dependencies needed for `spras` to run, it does not yet have a working installation of `spras` itself. -To install `spras` in the environment, finish by running the following from the root directory of the repository: -```bash -python -m pip install . -``` -Use caution when pip installing directly to your computer without using some form of virtual/conda environment as this can alter your system's underlying Python modules, which could lead to unexpected behavior. -In most cases, you should only `pip install` spras if you're already working in the `spras` conda environment! - -For developers, SPRAS can be installed via `pip` with the `-e` flag, as in `python -m pip install -e .`. This points Python back to the SPRAS repo so that any changes made to the source -code are reflected in the installed module. +The `spras` pixi environment comes bundled with all of Python dependencies needed for `spras` to run, including an installation of `spras` itself. You also need to install [Docker](https://docs.docker.com/get-docker/). After installing Docker, start Docker before running SPRAS. -Once you have activated the conda environment and started Docker, you can run SPRAS with the example Snakemake workflow. +Once you have activated the pixi environment and started Docker, you can run SPRAS with the example Snakemake workflow. From the root directory of the `spras` repository, run the command ``` snakemake --cores 1 --configfile config/config.yaml @@ -78,7 +67,7 @@ The Docker images are available on [DockerHub](https://hub.docker.com/orgs/reedc These wrappers are in the `spras/` subdirectory. **Test code**: Tests for the Docker wrappers and SPRAS code. -The tests require the conda environment in `environment.yml` and Docker. +The tests require the pixi environment via `pixi shell` and Docker. Run the tests with `pytest -s`. ## Singularity diff --git a/Snakefile b/Snakefile index 5b9340ccc..abf5fa5b2 100644 --- a/Snakefile +++ b/Snakefile @@ -12,8 +12,8 @@ import spras.config.config as _config SEP = '/' wildcard_constraints: - params="params-\w+", - dataset="\w+" + params=r"params-\w+", + dataset=r"\w+" # Elsewhere we import this as config, but in the Snakefile, the variable config is already populated # with the parsed config.yaml. This is done by Snakemake, which magically pipes config into this file diff --git a/docs/README.md b/docs/README.md index 678c770c9..6028b5066 100644 --- a/docs/README.md +++ b/docs/README.md @@ -45,10 +45,10 @@ these helpful tutorials that will prime you with some basics: ## Editing SPRAS Documentation -The `spras` conda environment currently comes pre-bundled with the tools you +The `spras` pixi environment currently comes pre-bundled with the tools you need to edit and build SPRAS's docs website, so the first step is making sure you have an active spras environment. If you don't, refer to this repo's -[root README.md](../README.md) for instructions on installing conda and +[root README.md](../README.md) for instructions on installing pixi and building/activating the `spras` env. ### Building And Viewing Your Changes diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst index 0d8a542ff..51920a194 100644 --- a/docs/contributing/index.rst +++ b/docs/contributing/index.rst @@ -215,9 +215,9 @@ Also test the functions available in the ``Dataset`` class. 0 True A Note: If you get a 'no module named' error, make sure that you are -running your interactive python session inside the SPRAS conda +running your interactive python session inside the SPRAS pixi environment (your terminal should begin with ``(spras)`` instead of -``(base)``, which can be done through ``conda activate spras``), and +``(base)``, which can be done through ``pixi shell``), and your editor's interpreter is set to using the SPRAS environment over the base environment (on VSCode and IntelliJ editors, this should be in the bottom right.) @@ -298,15 +298,7 @@ through SPRAS with snakemake --cores 1 --configfile config/config.yaml -Make sure to run the command inside the ``spras`` conda environment. - -If installing via ``pip`` instead of using conda, install with the -``-e .[dev]`` options (the full command to run from the repo root is -``python -m pip install -e .[dev]``) so that Python picks up any changes -you make and installs all optional development packages. Omitting the -``-e`` flag will prevent your changes from being reflected unless you -force re-install, and omitting ``.[dev]`` will prevent pip from -installing ``pre-commit`` and ``pytest``. +Make sure to run the command inside the ``spras`` pixi environment. As a workflow manager, Snakemake will consider the work described in the configuration file to be completed once the necessary output files have @@ -471,9 +463,8 @@ tests are run automatically on every commit through the GitHub Actions. However, developers will benefit from setting up their environment to run the same tests locally while they modify the SPRAS source. -The ``pre-commit`` package is installed as part of the conda environment -in ``environment.yml``, or when installing SPRAS with -``python -m pip install -e .[dev]``. From there, the pre-commit `quick +The ``pre-commit`` package is installed as part of the pixi environment +in ``environment.yml``. From there, the pre-commit `quick start `__ guide explains two primary ways to use it locally: diff --git a/docs/htcondor.rst b/docs/htcondor.rst index 4a96d4487..6e762af29 100644 --- a/docs/htcondor.rst +++ b/docs/htcondor.rst @@ -91,14 +91,14 @@ Submitting Parallel Jobs Parallelizing SPRAS workflows with HTCondor requires the same setup as the previous section, but with two additions. First, it requires an -activated SPRAS conda environment with a ``pip install``-ed version of +activated SPRAS pixi environment with a ``pip install``-ed version of the SPRAS module (via ``pip install .`` inside the SPRAS directory). Second, it requires an experimental executor for HTCondor that has been forked from the upstream `HTCondor Snakemake executor `__. -After activating your ``spras`` conda environment and ``pip``-installing +After activating your ``spras`` pixi environment and ``pip``-installing SPRAS, you can install the HTCondor Snakemake executor with the following: @@ -184,7 +184,7 @@ jobs where they left off. **Note**: If you encounter an error that says ``No module named 'spras'``, make sure you've ``pip install``-ed the -SPRAS module into your conda environment. +SPRAS module into your pixi environment. Job Monitoring -------------- @@ -246,9 +246,9 @@ containing: raise CredsError("Credentials not found for this workflow") it indicates you must upgrade the version of the HTCondor Snakemake -executor bundled with your conda environment. +executor bundled with your pixi environment. -To upgrade, from your activated ``spras`` conda environment run: +To upgrade, from your activated ``spras`` pixi environment run: .. code:: bash @@ -274,4 +274,4 @@ repository `__ and inspecting the commit history. If the preceding steps did not update the installed version, you may -need to delete and rebuild your ``spras`` conda environment. +need to delete and rebuild your ``spras`` pixi environment. diff --git a/docs/install.rst b/docs/install.rst index 6a68279b3..25fea3762 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -3,17 +3,15 @@ Installing SPRAS SPRAS provides several convenient ways to use the package, depending on your needs -Using a Conda Environment +Using Pixi ------------------------- -If you want to run spras locally on your machine, you can use the provided `environment.yml` file to create a conda environment -with all the necessary dependencies: -1. `Download and Install Conda`_ -2. Build the `spras` conda environment and activate it: +If you want to run spras locally on your machine, you can use ``pixi``: +1. `Download and Install Pixi`_ +2. Activate the pixi environment .. code-block:: bash - conda env create -f environment.yml - conda activate spras + pixi shell Installing SPRAS with `pip` --------------------------- @@ -38,4 +36,4 @@ If you want to pull a specific version of spras, use the version for the image t docker pull reedcompbio/spras:0.5.0 -.. _Download and Install Conda: https://conda-forge.org/download/ \ No newline at end of file +.. _Download and Install Pixi: https://pixi.sh/latest/installation/ diff --git a/docs/tutorial/beginner.rst b/docs/tutorial/beginner.rst index 9c8f7f236..d6f484faa 100644 --- a/docs/tutorial/beginner.rst +++ b/docs/tutorial/beginner.rst @@ -23,20 +23,14 @@ Visit the `SPRAS GitHub repository `__ an 0.2 Set up the SPRAS environment ------------------------------------- -From the root directory of the SPRAS repository, create and activate the Conda environment and install the SPRAS python package: +From the root directory of the SPRAS repository, create and activate the Pixi environment and install the SPRAS python package: .. code:: bash - conda env create -f environment.yml - conda activate spras - python -m pip install . + pixi shell .. note:: - The first command performs a one-time installation of the SPRAS dependencies by creating a Conda environment (an isolated space that keeps all required packages and versions separate from your system). - - The second command activates the newly created environment so you can use these dependencies when running SPRAS; this step must be done each time you open a new terminal session. - - The last command is a one-time installation of the SPRAS package into the environment. + This performs a one-time installation of the SPRAS dependencies by creating a Pixi environment (an isolated space that keeps all required packages and versions separate from your system). 0.3 Test the installation ------------------------- diff --git a/docs/tutorial/introduction.rst b/docs/tutorial/introduction.rst index 1484a896c..cdb7bb58d 100644 --- a/docs/tutorial/introduction.rst +++ b/docs/tutorial/introduction.rst @@ -18,7 +18,7 @@ Prerequisites for this tutorial =============================== Required software: -- `Conda `__ : for managing environments +- `Pixi `__ : for managing environments - `Docker `__ : for containerized runs - `Git `__: for cloning the SPRAS repository - A terminal or code editor (`VS Code `__ is recommended, but any terminal will work) diff --git a/docs/usage.rst b/docs/usage.rst index 0f60171ad..b8d0dd057 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -2,7 +2,7 @@ Using SPRAS =========== SPRAS is run through `Snakemake `_, which comes -with the SPRAS conda environment. +with the SPRAS pixi environment. To run SPRAS, run the following command inside the ``spras`` directory, specifying a ``config.yaml`` and the number of cores to run SPRAS with: diff --git a/environment.yml b/environment.yml deleted file mode 100644 index e5fc75b0c..000000000 --- a/environment.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: spras -channels: - - conda-forge -dependencies: - - adjusttext=1.3.0 - - bioconda::snakemake-minimal=9.6.2 - # Conda refers to pypi/docker as docker-py. - - docker-py=7.1.0 - - matplotlib=3.10.3 - - networkx=3.5 - - pandas=2.3.0 - - pydantic=2.11.7 - - numpy=2.3.1 - - requests=2.32.4 - - scikit-learn=1.7.0 - - seaborn=0.13.2 - - spython=0.3.14 - - # conda-specific for dsub - - python-dateutil=2.9.0 - - pytz=2025.2 - - pyyaml=6.0.2 - - tenacity=9.1.2 - - tabulate=0.9.0 - - # toolchain deps - - pip=25.3 - # This should be the same as requires-python minus the >=. - - python=3.11 - - # development dependencies - - pre-commit=4.2.0 - - pytest=8.4.1 - - pytest-split=0.10.0 - - - pip: - - dsub==0.4.13 diff --git a/pyproject.toml b/pyproject.toml index 18a6158cd..f953b31da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,3 +81,16 @@ include = ["spras*"] # packages tells setuptools what the exported package is called (ie allows import spras) packages = ["spras", "spras.analysis", "spras.config"] + +[tool.pixi.project] +channels = ["conda-forge", "bioconda"] +platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"] + +[tool.pixi.dependencies] +# for yamlfmt +go = ">=1.24.4,<2" +# to force local usage of the oldest supported python ver. +python = "3.11.*" + +[tool.pixi.environments] +default = {features = ["dev"], solve-group = "default"} diff --git a/spras/__init__.py b/spras/__init__.py index 50807dafe..88df06442 100644 --- a/spras/__init__.py +++ b/spras/__init__.py @@ -8,7 +8,7 @@ # Import version info from package metadata, which is populated from pyproject.toml # Note that this version will only be populated correctly in source code if the `spras` module -# is installed (as opposed to operating solely from a conda env). +# is installed (as opposed to operating solely from a pixi env). try: __version__ = version("spras") except PackageNotFoundError: