Reorganize forcefield tests

.github/workflows/testing.yml

@@ -23,7 +23,7 @@ jobs:
 
       - uses: pre-commit/action@v3.0.0
 
-  test-non-ase:
+  test-forcefields:
     # prevent this action from running on forks
     if: github.repository == 'materialsproject/atomate2'
 
@@ -39,8 +39,8 @@ jobs:
         shell: bash -l {0} # enables conda/mamba env activation by reading bash profile
     strategy:
       matrix:
-        python-version: ["3.11", "3.12"]
-        split: [1, 2, 3]
+        python-version: ["3.12"]
+        dep-group: ["generic", "torch-limited", "e3nn-limited"]
 
     steps:
       - name: Check out repo
@@ -59,17 +59,15 @@ jobs:
 
       - name: Install conda dependencies
         run: |
-          micromamba install -n a2 -c conda-forge enumlib packmol bader --yes
+          micromamba install -n a2 -c conda-forge packmol --yes
 
       - name: Install dependencies
-        run: |
+        run: | # TODO: remove setuptools pin once `mattersim` removes `pkg_resources` dependency
           micromamba activate a2
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade pip 'setuptools<81'
           mkdir -p ~/.abinit/pseudos
           cp -r tests/test_data/abinit/pseudos/ONCVPSP-PBE-SR-PDv0.4 ~/.abinit/pseudos
-          uv pip install .[strict,strict-forcefields,abinit,approxneb,aims] --group tests
-          uv pip install torch-runstats torch_dftd
-          uv pip install --no-deps nequip==0.5.6
+          uv pip install .[strict,strict-forcefields-${{ matrix.dep-group }},abinit,approxneb,aims] --group tests
 
       - name: Install pymatgen from master if triggered by pymatgen repo dispatch
         if: github.event_name == 'repository_dispatch' && github.event.action == 'pymatgen-ci-trigger'
@@ -81,15 +79,17 @@ jobs:
         env:
           MP_API_KEY: ${{ secrets.MP_API_KEY }}
 
-        # regenerate durations file with `pytest --store-durations --durations-path tests/.pytest-split-durations`
-        # Note the use of `--splitting-algorithm least_duration`.
-        # This helps prevent a test split having no tests to run, and then the GH action failing, see:
-        # https://github.com/jerry-git/pytest-split/issues/95
-        # However this `splitting-algorithm` means that tests cannot depend sensitively on the order they're executed in.
         run: |
           micromamba activate a2
-          pytest -n auto --splits 3 --group ${{ matrix.split }} --durations-path tests/.pytest-split-durations --splitting-algorithm least_duration --ignore=tests/ase --ignore=tests/openff_md --ignore=tests/openmm_md --cov=atomate2 --cov-report=xml
+          pytest -n auto --cov=atomate2 --cov-report=xml tests/forcefields
 
+      - name: Forcefield tutorial
+        if: matrix.dep-group == 'torch-limited'
+        env:
+          MP_API_KEY: ${{ secrets.MP_API_KEY }}
+        run: |
+          micromamba activate a2
+          pytest -n auto --nbmake ./tutorials/force_fields
 
       - uses: codecov/codecov-action@v1
         if: matrix.python-version == '3.11' && github.repository == 'materialsproject/atomate2'
@@ -98,7 +98,7 @@ jobs:
           name: coverage${{ matrix.split }}
           file: ./coverage.xml
 
-  test-openff:
+  test-non-ase:
     # prevent this action from running on forks
     if: github.repository == 'materialsproject/atomate2'
 
@@ -114,7 +114,8 @@ jobs:
         shell: bash -l {0} # enables conda/mamba env activation by reading bash profile
     strategy:
       matrix:
-        python-version: ["3.11","3.12"]
+        python-version: ["3.11", "3.12"]
+        split: [1, 2, 3]
 
     steps:
       - name: Check out repo
@@ -132,14 +133,18 @@ jobs:
         run: micromamba run -n a2 pip install uv
 
       - name: Install conda dependencies
-        run: |  # TODO: migrate openff tests to use non smirnoff99frosst forcefields - requires old setuptools / pkg_resources
-          micromamba install -n a2 -c conda-forge enumlib packmol bader openbabel openff-toolkit==0.16.2 openff-interchange==0.3.22 'setuptools<81' --yes
+        run: |
+          micromamba install -n a2 -c conda-forge enumlib packmol bader --yes
 
       - name: Install dependencies
         run: |
           micromamba activate a2
           python -m pip install --upgrade pip
-          uv pip install .[strict-openff] --group tests
+          mkdir -p ~/.abinit/pseudos
+          cp -r tests/test_data/abinit/pseudos/ONCVPSP-PBE-SR-PDv0.4 ~/.abinit/pseudos
+          uv pip install .[strict,abinit,approxneb,aims] --group tests
+          uv pip install torch-runstats torch_dftd
+          uv pip install --no-deps nequip==0.5.6
 
       - name: Install pymatgen from master if triggered by pymatgen repo dispatch
         if: github.event_name == 'repository_dispatch' && github.event.action == 'pymatgen-ci-trigger'
@@ -151,19 +156,27 @@ jobs:
         env:
           MP_API_KEY: ${{ secrets.MP_API_KEY }}
 
+        # regenerate durations file with `pytest --store-durations --durations-path tests/.pytest-split-durations`
+        # Note the use of `--splitting-algorithm least_duration`.
+        # This helps prevent a test split having no tests to run, and then the GH action failing, see:
+        # https://github.com/jerry-git/pytest-split/issues/95
+        # However this `splitting-algorithm` means that tests cannot depend sensitively on the order they're executed in.
         run: |
           micromamba activate a2
-          pytest -n auto tests/{openff_md,openmm_md}
+          pytest -n auto --splits 3 --group ${{ matrix.split }} --durations-path tests/.pytest-split-durations --splitting-algorithm least_duration --ignore=tests/ase --ignore=tests/openff_md --ignore=tests/openmm_md --ignore=tests/forcefields --cov=atomate2 --cov-report=xml
 
-  test-notebooks-and-ase:
+
+      - uses: codecov/codecov-action@v1
+        if: matrix.python-version == '3.11' && github.repository == 'materialsproject/atomate2'
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          name: coverage${{ matrix.split }}
+          file: ./coverage.xml
+
+  test-openff:
     # prevent this action from running on forks
     if: github.repository == 'materialsproject/atomate2'
 
-    # It seems like anything torch-dependent and tblite can't be installed in the same environment
-    # without the tblite tests failing in CI, see, e.g.:
-    # https://github.com/tblite/tblite/issues/116
-    # Outside of CI, having torch installed but not loaded seems not to affect tblite
-    # Set off ASE tests here, where tblite-dependent tests live
     services:
       local_mongodb:
         image: mongo:4.0
@@ -176,7 +189,7 @@ jobs:
         shell: bash -l {0} # enables conda/mamba env activation by reading bash profile
     strategy:
       matrix:
-        python-version: ["3.11", "3.12"]
+        python-version: ["3.11","3.12"]
 
     steps:
       - name: Check out repo
@@ -194,46 +207,38 @@ jobs:
         run: micromamba run -n a2 pip install uv
 
       - name: Install conda dependencies
-        run: |
-          micromamba install -n a2 -c conda-forge enumlib packmol openbabel --yes
+        run: |  # TODO: migrate openff tests to use non smirnoff99frosst forcefields - requires old setuptools / pkg_resources
+          micromamba install -n a2 -c conda-forge enumlib packmol bader openbabel openff-toolkit==0.16.2 openff-interchange==0.3.22 'setuptools<81' --yes
 
       - name: Install dependencies
         run: |
           micromamba activate a2
           python -m pip install --upgrade pip
-          uv pip install .[strict,aims] --group tests
-          uv pip install tblite>=0.4.0
+          uv pip install .[strict-openff] --group tests
 
       - name: Install pymatgen from master if triggered by pymatgen repo dispatch
         if: github.event_name == 'repository_dispatch' && github.event.action == 'pymatgen-ci-trigger'
         run: |
           micromamba activate a2
           uv pip install --upgrade 'git+https://github.com/materialsproject/pymatgen@${{ github.event.client_payload.pymatgen_ref }}'
 
-      - name: Test Notebooks
+      - name: Test split ${{ matrix.split }}
         env:
           MP_API_KEY: ${{ secrets.MP_API_KEY }}
-        run: |
-          micromamba activate a2
-          pytest -n auto --nbmake ./tutorials --ignore=./tutorials/openmm_tutorial.ipynb --ignore=./tutorials/force_fields --ignore=./tutorials/torchsim_tutorial.ipynb
 
-      - name: Test ASE
-        env:
-          MP_API_KEY: ${{ secrets.MP_API_KEY }}
         run: |
           micromamba activate a2
-          pytest -n auto --splits 1 --group 1 --cov=atomate2 --cov-report=xml tests/ase
-
-      - uses: codecov/codecov-action@v1
-        if: matrix.python-version == '3.11' && github.repository == 'materialsproject/atomate2'
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          file: ./coverage.xml
+          pytest -n auto tests/{openff_md,openmm_md}
 
-  test-force-field-notebook:
+  test-notebooks-and-ase:
     # prevent this action from running on forks
     if: github.repository == 'materialsproject/atomate2'
 
+    # It seems like anything torch-dependent and tblite can't be installed in the same environment
+    # without the tblite tests failing in CI, see, e.g.:
+    # https://github.com/tblite/tblite/issues/116
+    # Outside of CI, having torch installed but not loaded seems not to affect tblite
+    # Set off ASE tests here, where tblite-dependent tests live
     services:
       local_mongodb:
         image: mongo:4.0
@@ -263,45 +268,43 @@ jobs:
       - name: Install uv
         run: micromamba run -n a2 pip install uv
 
+      - name: Install conda dependencies
+        run: |
+          micromamba install -n a2 -c conda-forge enumlib packmol openbabel --yes
+
       - name: Install dependencies
         run: |
           micromamba activate a2
           python -m pip install --upgrade pip
-          mkdir -p ~/.abinit/pseudos
-          cp -r tests/test_data/abinit/pseudos/ONCVPSP-PBE-SR-PDv0.4 ~/.abinit/pseudos
-          uv pip install .[strict,strict-forcefields,abinit,aims] --group tests
-          uv pip install torch-runstats
-          uv pip install --no-deps nequip==0.5.6
+          uv pip install .[strict,aims] --group tests
+          uv pip install tblite>=0.4.0
 
       - name: Install pymatgen from master if triggered by pymatgen repo dispatch
         if: github.event_name == 'repository_dispatch' && github.event.action == 'pymatgen-ci-trigger'
         run: |
           micromamba activate a2
           uv pip install --upgrade 'git+https://github.com/materialsproject/pymatgen@${{ github.event.client_payload.pymatgen_ref }}'
 
-      - name: Forcefield tutorial
+      - name: Test Notebooks
         env:
           MP_API_KEY: ${{ secrets.MP_API_KEY }}
-
-        # regenerate durations file with `pytest --store-durations --durations-path tests/.pytest-split-durations`
-        # Note the use of `--splitting-algorithm least_duration`.
-        # This helps prevent a test split having no tests to run, and then the GH action failing, see:
-        # https://github.com/jerry-git/pytest-split/issues/95
-        # However this `splitting-algorithm` means that tests cannot depend sensitively on the order they're executed in.
         run: |
           micromamba activate a2
-          pytest -n auto --nbmake ./tutorials/force_fields
+          pytest -n auto --nbmake ./tutorials --ignore=./tutorials/openmm_tutorial.ipynb --ignore=./tutorials/force_fields --ignore=./tutorials/torchsim_tutorial.ipynb
 
+      - name: Test ASE
+        env:
+          MP_API_KEY: ${{ secrets.MP_API_KEY }}
+        run: |
+          micromamba activate a2
+          pytest -n auto --splits 1 --group 1 --cov=atomate2 --cov-report=xml tests/ase
 
       - uses: codecov/codecov-action@v1
         if: matrix.python-version == '3.11' && github.repository == 'materialsproject/atomate2'
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          name: coverage
           file: ./coverage.xml
 
-
-
   test-torchsim:
     # prevent this action from running on forks
     if: github.repository == 'materialsproject/atomate2'
@@ -373,15 +376,15 @@ jobs:
           cache-dependency-path: pyproject.toml
 
       - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install .[strict,strict-forcefields] --group docs
+        run: | # TODO: remove setuptools pin once `mattersim` removes `pkg_resources` dependency
+          python -m pip install --upgrade pip 'setuptools<81'
+          pip install .[strict,strict-forcefields-generic] --group docs
 
       - name: Build
         run: sphinx-build docs docs_build
 
   automerge:
-    needs: [docs, lint, test-force-field-notebook, test-non-ase, test-notebooks-and-ase, test-openff, test-torchsim]
+    needs: [docs, lint, test-forcefields, test-non-ase, test-notebooks-and-ase, test-openff, test-torchsim]
     runs-on: ubuntu-latest
 
     permissions:

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ default_language_version:
 exclude: ^(.github/|tests/test_data/abinit/)
 repos:
 - repo: https://github.com/charliermarsh/ruff-pre-commit
-  rev: v0.15.0
+  rev: v0.15.1
   hooks:
   - id: ruff
     args: [--fix]

docs/dev/forcefields.md

@@ -0,0 +1,30 @@
+# Maintaining the machine learning forcefields module
+
+Some of these points are already noted in the `pyproject.toml`. This goes into a bit more depth.
+
+## Overview
+
+`atomate2` contains a convenience interface to many common machine learning interatomic forcefields (MLFFs) via the atomic simulation environment (ASE). In the literature, these may also be known as machine learning interatomic potentials (MLIPs), or, when specifically referring to MLIPs with coverage of most of the periodic table, foundation potentials (FPs).
+
+There is both an `ase` module in `atomate2`, based around general `ase` `Calculator`s, and a `forcefields`-specific module which has a higher number of workflows.
+
+The `ase` module should be used to manage high-level tasks, such as geometry optimization, molecular dynamics, and nudged elastic band. Any further developments to these tools in `ase` should also warrant updates in this module in `atomate2`. For example, when `ase` rolled out the `MTKNPT` NPT MD barostat as a replacement for the default barostat, this was also made the default in `atomate2`.
+
+The `forcefields` library should be used to develop concrete implementations of workflows, e.g., harmonic phonon, Grüneisen parameter, ApproxNEB.
+
+## Dependency Chaos
+
+The individual MLFFs in `atomate2` often have conflicting dependencies. This makes testing and managing a consistent, relatively up-to-date testing environment challenging.
+We want to avoid pinning MLFF libraries at older versions, because this may break their API within `atomate2`, or lead to drift in test data as models evolve.
+
+Thus, it is likely that the `pyproject.toml` contains multiple optional dependencies under the header `strict-forcefields-*`. These groupings are used to ensure the most recent version of a MLFF library is installed in CI testing, with acceptable dependencies. The names of these groups can change over time, but the names should be chosen to be informative as to why they exist: ex., `strict-forcefields-e3nn-limited` to indicate that these MLFFs need an older version of `e3nn`, or `strict-forcefields-generic` to indicate that no strong dependency limitation is observed.
+
+When updating these groupings, it is critical to ensure that you also update the `.github/workflows/testing.yml` testing workflow. You will see that the different forcefield dependency groups are tested separately.
+
+When adding a new MLFF and tests for it (if possible), you must ensure that appropriate `pytest.mark.skipif` decorators are applied if that MLFF package is not installed. A `mlff_is_installed` boolean check is included in `tests/forcefields/conftest.py` for convenience in writing these skip test markers. See `tests/forcefields/test_jobs.py` for examples.
+
+## Testing limitations
+
+Some MLFFs, like FAIRChem, have access restrictions on them which prohibit running tests in CI. For these, we should also likely create tests for continuing development even if they are not run in CI.
+
+Other MLFFs, like GAP or Nequip, are more generic architectures and require specific potential files to describe certain chemical spaces. Contributors adding new architectures which require these potential fields should submit minimal potential files (as small as possible to test, accuracy is not important here) to run tests for these.

docs/dev/input_sets.md

@@ -11,7 +11,7 @@ Most ab-initio codes rely on input files read from disk. The `InputSet` class au
 - As a dictionary-like collection, the `InputSet` associates file names (the keys) with their content (which can be either simple strings or `InputFile` instances). The main methods of this class are `write_input()` for standardized file writing and `from_directory()` to read in pre-existing `InputSets`. The `validate()` method confirms the validity of the `InputSet`.
 - `InputGenerators` implement the `get_input_set()` method, which provides the recipe, i.e., logic to return a suitable `InputSet` for a `Molecule`, `Structure`, or another set of atomic coordinates. During the initialization of an `InputGenerator` additional inputs can be provided that adjust the recipe.
 
-While generally the input classes are supposed to be part of `pymatgen` during development it is recommended to include them in `atomate2` at first to facilitate rapid iteration. Once mature, they can be moved to `pymatgen` or to a `pymatgen` [addon package](https://.org/addons). When implementing your own input classes take note of the recommendations and rules in the pymatgen documentation [[1](https://pymatgen.org/pymatgen.io.html#module-pymatgen.io.core), [2](https://pymatgen.org/pymatgen.io.vasp.html#module-pymatgen.io.vasp.sets)].
+While generally the input classes are supposed to be part of `pymatgen` during development it is recommended to include them in `atomate2` at first to facilitate rapid iteration. Once mature, they can be moved to `pymatgen` or to a `pymatgen` [addon package](https://github.com/materialsproject/pymatgen-addon-template). When implementing your own input classes take note of the recommendations and rules in the pymatgen documentation [[1](https://pymatgen.org/pymatgen.io.html#module-pymatgen.io.core), [2](https://pymatgen.org/pymatgen.io.vasp.html#module-pymatgen.io.vasp.sets)].
 
 The `InputGenerators` interact with the atomate2 workflows through `Makers`. Each `Maker` for a code that requires input files, sets its `input_set_generator` parameter during initialization. For example, for the `RelaxMaker`, the default `input_set_generator` is the `RelaxSetGenerator`, but of course, users can provide or modify any `VaspInputGenerator` and provide it as input to a Maker.
 

docs/index.md

@@ -10,6 +10,7 @@ user/jobflow-remote
 user/fireworks
 user/atomate-1-vs-2
 user/codes/index
+user/addons
 tutorials/tutorials
 ```
 
@@ -26,6 +27,7 @@ dev/dev_install
 dev/workflow_tutorial
 dev/vasp_tests
 dev/abinit_tests
+dev/forcefields
 ```
 
 ```{toctree}