Cache macos/ubuntu optional dependency install in CI pipeline, enable mcsqs/pyzeo tests (#4455)

* separate optional dep install script

* disable wget cert check for now

* try dependency cache

* optional dep needed before pyzeo install

* Revert "optional dep needed before pyzeo install"

This reverts commit 10543b9.

* install zeopp in optional dep

* skip pyzeo on windows

* tyep annotation

* install mcsqs

* fix mcsqs runtime error

* fix mcsqs caller test

* declare global opt bin dir

* skip failing test for now

* reuse zeo_found

* more readable cache key

* pytext skip -> xfail

* fix test class naming

* remove unnecessary test file dir check

Author: DanielYang59

.github/workflows/_install_opt_unit_deps.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+# Use a custom bin directory for easier GitHub Actions caching
+BIN_DIR="${1:-$PWD/opt/bin}" && mkdir -p "$BIN_DIR"
+
+# Install BoltzTraP
+wget --no-verbose -O BoltzTraP.tar.bz2 https://owncloud.tuwien.ac.at/index.php/s/s2d55LYlZnioa3s/download
+tar -jxf BoltzTraP.tar.bz2
+cp boltztrap-1.2.5/src/{BoltzTraP,x_trans} "$BIN_DIR"
+
+# Install Vampire 5.0
+# TODO: Accepts self-signed cert (https://github.com/richard-evans/vampire/issues/122)
+wget --no-verbose --no-check-certificate https://vampire.york.ac.uk/resources/release-5/vampire-5.0-linux.tar.gz
+tar -zxf vampire-5.0-linux.tar.gz && mv linux vampire-5.0
+cp vampire-5.0/vampire-serial "$BIN_DIR"
+
+# Install Voro++ and ZEO++
+wget --no-verbose http://www.zeoplusplus.org/zeo++-0.3.tar.gz
+tar -xzf zeo++-0.3.tar.gz
+make -C zeo++-0.3/voro++/src -s CFLAGS="-w"
+make -C zeo++-0.3 -s CFLAGS="-w"
+cp zeo++-0.3/voro++/src/voro++ "$BIN_DIR"
+cp zeo++-0.3/network "$BIN_DIR"
+
+# Install mcsqs (from ATAT)
+# TODO: cannot build on MacOS
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+  wget --no-verbose https://axelvandewalle.github.io/www-avdw/atat/atat3_50.tar.gz
+  tar -zxf atat3_50.tar.gz
+
+  # Replace `BINDIR` in makefile
+  sed -i "s|^BINDIR=.*|BINDIR=$BIN_DIR|" atat/makefile
+
+  sudo apt install csh -y -q
+  make -C atat && make -C atat install
+fi
+
+# Add to path
+echo "$BIN_DIR" >> "$GITHUB_PATH"

.github/workflows/test.yml

@@ -56,6 +56,7 @@ jobs:
       MPLBACKEND: Agg  # non-interactive backend for matplotlib
       PMG_MAPI_KEY: ${{ secrets.PMG_MAPI_KEY }}
       PYTHONWARNDEFAULTENCODING: "true"  # PEP 597: Enable optional EncodingWarning
+      OPT_BIN_DIR: ${{ github.workspace }}/opt/bin  # for optional Unix dependencies
     steps:
       - name: Check out repo
         uses: actions/checkout@v4
@@ -94,23 +95,17 @@ jobs:
           pip install --upgrade matgl>=1.2.6
           pip install torch==2.2.0 torchdata==0.7.1
 
-      - name: Install optional Ubuntu and macOS dependencies
+      - name: Restore cache for optional Ubuntu/MacOS dependencies
         if: matrix.config.os == 'ubuntu-latest' || matrix.config.os == 'macos-latest'
-        run: |
-          # Install BoltzTraP
-          wget -q -O BoltzTraP.tar.bz2 https://owncloud.tuwien.ac.at/index.php/s/s2d55LYlZnioa3s/download
-          tar -jxf BoltzTraP.tar.bz2 && realpath boltztrap-1.2.5/src/ >> $GITHUB_PATH
-
-          # Install Vampire 5.0
-          wget -q https://vampire.york.ac.uk/resources/release-5/vampire-5.0-linux.tar.gz
-          tar -zxf vampire-5.0-linux.tar.gz && mv linux vampire-5.0
-          realpath vampire-5.0/ >> $GITHUB_PATH
-
-          # Install Voro++ and ZEO++
-          wget -q http://www.zeoplusplus.org/zeo++-0.3.tar.gz && tar -xzf zeo++-0.3.tar.gz
+        id: optbin-cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.OPT_BIN_DIR }}
+          key: opt-bin-${{ runner.os }}-${{ hashFiles('.github/workflows/_install_opt_unit_deps.sh') }}
 
-          cd zeo++-0.3/voro++/src && make -s CFLAGS="-w" && realpath . >> $GITHUB_PATH && cd ../..
-          make -s CFLAGS="-w" && realpath . >> $GITHUB_PATH
+      - name: Build optional Ubuntu/MacOS dependencies (when cache misses)
+        if: (matrix.config.os == 'ubuntu-latest' || matrix.config.os == 'macos-latest') && steps.optbin-cache.outputs.cache-hit != 'true'
+        run: bash .github/workflows/_install_opt_unit_deps.sh "$OPT_BIN_DIR"
 
       - name: pytest split ${{ matrix.split }}
         env:

pyproject.toml

@@ -105,7 +105,7 @@ mlp = ["matgl>=1.2.7 ; python_version<'3.13'"]
 numba = ["numba>=0.55"]
 numpy-v1 = ["numpy>=1.25.0,<2"] # Test NP1 on Windows (quite buggy ATM)
 optional = [
-    "pymatgen[abinit,ase,mlp,tblite,matcalc]",
+    "pymatgen[abinit,ase,matcalc,mlp,tblite,zeopp]",
     "beautifulsoup4",
     # PR4347: BoltzTraP2 needs to bump cmake version for spglib build
     # "BoltzTraP2>=24.9.4",
@@ -125,7 +125,7 @@ symmetry = ["moyopy[interface]>=0.3", "spglib>=2.5"]
 # https://github.com/tblite/tblite/issues/175
 tblite = ["tblite[ase]>=0.3.0; platform_system=='Linux' and python_version<'3.12'"]
 vis = ["vtk>=6.0.0"]
-zeopp = ["pyzeo"]  # Note: requires Voro++ and Zeo++ to be installed
+zeopp = ["pyzeo; platform_system != 'Windows'"]  # Note: requires Voro++ and Zeo++ to be installed
 
 [project.scripts]
 pmg = "pymatgen.cli.pmg:main"

src/pymatgen/command_line/mcsqs_caller.py

@@ -146,7 +146,7 @@ def run_mcsqs(
 
         raise RuntimeError("mcsqs exited before timeout reached")
 
-    except TimeoutExpired:
+    except TimeoutExpired as exc:
         for process in mcsqs_find_sqs_processes:
             process.kill()
             process.communicate()
@@ -157,7 +157,7 @@ def run_mcsqs(
                 raise RuntimeError(
                     "mcsqs did not generate output files, "
                     "is search_time sufficient or are number of instances too high?"
-                )
+                ) from exc
 
             process = Popen(["mcsqs", "-best"])
             process.communicate()
@@ -166,7 +166,7 @@ def run_mcsqs(
             return _parse_sqs_path(".")
 
         os.chdir(original_directory)
-        raise TimeoutError("Cluster expansion took too long.")
+        raise TimeoutError("Cluster expansion took too long.") from exc
 
 
 def _parse_sqs_path(path) -> Sqs:

src/pymatgen/io/zeopp.py

@@ -41,8 +41,8 @@
     from zeo.cluster import prune_voronoi_network_close_node
     from zeo.netstorage import AtomNetwork
 
-    zeo_found = True
-    zeo_source: str | None = "zeo"
+    zeo_found: bool = True
+    zeo_source: Literal["zeo", "pyzeo"] | None = "zeo"
 except ImportError:
     try:
         from pyzeo import AtomNetwork
@@ -58,6 +58,7 @@
 
 if TYPE_CHECKING:
     from pathlib import Path
+    from typing import Literal
 
     from typing_extensions import Self
 

tests/analysis/chemenv/coordination_environments/test_coordination_geometry_finder.py

@@ -166,12 +166,12 @@ def _strategy_test(self, strategy):
                     # Check that the environment found is the expected one
                     assert coord_env == expected_geoms[ienv]
 
-    @pytest.mark.skip("TODO: need someone to fix this")
+    @pytest.mark.xfail(reason="TODO: need someone to fix this")
     def test_simplest_chemenv_strategy(self):
         strategy = SimplestChemenvStrategy()
         self._strategy_test(strategy)
 
-    @pytest.mark.skip("TODO: need someone to fix this")
+    @pytest.mark.xfail(reason="TODO: need someone to fix this")
     def test_simple_abundance_chemenv_strategy(self):
         strategy = SimpleAbundanceChemenvStrategy()
         self._strategy_test(strategy)

tests/analysis/test_graphs.py

@@ -239,7 +239,7 @@ def test_auto_image_detection(self):
 
         assert len(list(struct_graph.graph.edges(data=True))) == 3
 
-    @pytest.mark.skip(reason="Need someone to fix this, see issue 4206")
+    @pytest.mark.xfail(reason="Need someone to fix this, see issue 4206")
     def test_str(self):
         square_sg_str_ref = """Structure Graph
 Structure:

tests/analysis/test_molecule_structure_comparator.py

@@ -139,7 +139,7 @@ def test_to_and_from_dict(self):
         d2 = MoleculeStructureComparator.from_dict(d1).as_dict()
         assert d1 == d2
 
-    @pytest.mark.skip("TODO: need someone to fix this")
+    @pytest.mark.xfail(reason="TODO: need someone to fix this")
     def test_structural_change_in_geom_opt(self):
         qcout_path = f"{TEST_DIR}/mol_1_3_bond.qcout"
         qcout = QCOutput(qcout_path)

tests/analysis/test_piezo_sensitivity.py

@@ -208,7 +208,7 @@ def test_get_stable_fcm(self):
             assert_allclose(asum1, np.zeros([3, 3]), atol=1e-5)
             assert_allclose(asum2, np.zeros([3, 3]), atol=1e-5)
 
-    @pytest.mark.skipif(
+    @pytest.mark.xfail(
         platform.system() == "Windows" and int(np.__version__[0]) >= 2,
         reason="See https://github.com/conda-forge/phonopy-feedstock/pull/158#issuecomment-2227506701",
     )
@@ -262,7 +262,7 @@ def test_get_piezo(self):
         piezo = get_piezo(self.BEC, self.IST, self.FCM)
         assert_allclose(piezo, self.piezo, atol=1e-5)
 
-    @pytest.mark.skipif(
+    @pytest.mark.xfail(
         platform.system() == "Windows" and int(np.__version__[0]) >= 2,
         reason="See https://github.com/conda-forge/phonopy-feedstock/pull/158#issuecomment-2227506701",
     )

tests/analysis/test_surface_analysis.py

@@ -253,37 +253,37 @@ def test_entry_dict_from_list(self):
         surf_ene_plotter = SurfaceEnergyPlotter(all_Pt_slab_entries, self.Pt_analyzer.ucell_entry)
         assert surf_ene_plotter.list_of_chempots == self.Pt_analyzer.list_of_chempots
 
-    @pytest.mark.skip("TODO: need someone to fix this")
+    @pytest.mark.xfail(reason="TODO: need someone to fix this")
     def test_monolayer_vs_BE(self):
         for el in self.Oads_analyzer_dict:
             # Test WulffShape for adsorbed surfaces
             analyzer = self.Oads_analyzer_dict[el]
             analyzer.monolayer_vs_BE()
 
-    @pytest.mark.skip("TODO: need someone to fix this")
+    @pytest.mark.xfail(reason="TODO: need someone to fix this")
     def test_area_frac_vs_chempot_plot(self):
         for el in self.Oads_analyzer_dict:
             # Test WulffShape for adsorbed surfaces
             analyzer = self.Oads_analyzer_dict[el]
             analyzer.area_frac_vs_chempot_plot(x_is_u_ads=True)
 
-    @pytest.mark.skip("TODO: need someone to fix this")
+    @pytest.mark.xfail(reason="TODO: need someone to fix this")
     def test_chempot_vs_gamma_clean(self):
         self.Cu_analyzer.chempot_vs_gamma_clean()
         for el in self.Oads_analyzer_dict:
             # Test WulffShape for adsorbed surfaces
             analyzer = self.Oads_analyzer_dict[el]
             analyzer.chempot_vs_gamma_clean(x_is_u_ads=True)
 
-    @pytest.mark.skip("TODO: need someone to fix this")
+    @pytest.mark.xfail(reason="TODO: need someone to fix this")
     def test_chempot_vs_gamma_facet(self):
         for el, val in self.metals_O_entry_dict.items():
             for hkl in val:
                 # Test WulffShape for adsorbed surfaces
                 analyzer = self.Oads_analyzer_dict[el]
                 analyzer.chempot_vs_gamma_facet(hkl)
 
-    @pytest.mark.skip("TODO: need someone to fix this")
+    @pytest.mark.xfail(reason="TODO: need someone to fix this")
     def test_surface_chempot_range_map(self):
         for el, val in self.metals_O_entry_dict.items():
             for hkl in val: