Merge pull request #463 from minrk/cluster-manager

Cluster Python API

Author: minrk

.github/workflows/test.yml

@@ -46,15 +46,36 @@ jobs:
           - python: "3.6"
           - python: "3.7"
           - python: "3.8"
+            mpi: true
           - python: "3.9"
 
     steps:
       - uses: actions/checkout@v2
 
+      - name: Cache conda environment
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda
+          key: conda
+
+      - name: Install Python (conda) ${{ matrix.python }}
+        if: matrix.mpi
+        run: |
+          export MAMBA_ROOT_PREFIX=$HOME/conda
+          test -d $MAMBA_ROOT_PREFIX || mkdir $MAMBA_ROOT_PREFIX
+          wget -qO- https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba
+          eval "$(./bin/micromamba shell hook -s posix)"
+          micromamba activate
+          micromamba install -y -c conda-forge mpich mpi4py python=${{ matrix.python }}
+          echo "PATH=$MAMBA_ROOT_PREFIX/bin:$PATH" >> $GITHUB_ENV
+
       - name: Install Python ${{ matrix.python }}
+        if: ${{ ! matrix.mpi }}
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python }}
+
       - name: Install Python dependencies
         run: |
           pip install --upgrade pip
@@ -67,11 +88,18 @@ jobs:
       - name: Show environment
         run: pip freeze
 
+      - name: Run mpi tests
+        if: ${{ matrix.mpi }}
+        run: |
+          pytest -vx --color=yes --cov=ipyparallel ipyparallel/tests/test_cluster.py
+
       - name: Run tests
+        if: ${{ ! matrix.mpi }}
         # FIXME: --color=yes explicitly set because:
         #        https://github.com/actions/runner/issues/241
         run: |
           pytest -v --maxfail=3 --color=yes --cov=ipyparallel ipyparallel/tests
+
       - name: Submit codecov report
         run: |
           codecov

.gitignore

@@ -22,3 +22,4 @@ __pycache__
 *coverage.xml
 .coverage.*
 .idea
+htmlcov

docs/source/api/ipyparallel.rst

@@ -11,6 +11,8 @@ API Reference
 Classes
 -------
 
+.. autoclass:: Cluster
+
 .. autoclass:: Client
 
 .. autoclass:: DirectView

docs/source/conf.py

@@ -60,13 +60,13 @@
 # You can specify multiple suffix as a list of string:
 source_suffix = ['.rst', '.md']
 
-from traitlets.config import HasTraits
+from traitlets.config import LoggingConfigurable
 
 # exclude members inherited from HasTraits by default
 autodoc_default_options = {
     'members': None,
     "inherited-members": None,
-    "exclude-members": ','.join(dir(HasTraits)),
+    "exclude-members": ','.join(dir(LoggingConfigurable)),
 }
 
 # Add dev disclaimer.

docs/source/tutorial/process.rst

@@ -165,7 +165,7 @@ and Windows HPC Server.
 
 In general, these are configured by the :attr:`IPClusterEngines.engine_set_launcher_class`,
 and :attr:`IPClusterStart.controller_launcher_class` configurables, which can be the
-fully specified object name (e.g. ``'ipyparallel.apps.launcher.LocalControllerLauncher'``),
+fully specified object name (e.g. ``'ipyparallel.cluster.launcher.LocalControllerLauncher'``),
 but if you are using IPython's builtin launchers, you can specify a launcher by its prefix e.g:
 
 .. sourcecode:: python
@@ -174,7 +174,7 @@ but if you are using IPython's builtin launchers, you can specify a launcher by
     # equivalent to
     c.IPClusterEngines.engine_launcher_class = 'SSHEngineSetLauncher'
     # both of which expand to
-    c.IPClusterEngines.engine_launcher_class = 'ipyparallel.apps.launcher.SSHEngineSetLauncher'
+    c.IPClusterEngines.engine_launcher_class = 'ipyparallel.cluster.launcher.SSHEngineSetLauncher'
 
 The shortest form being of particular use on the command line, where all you need to do to
 get an IPython cluster running with engines started with MPI is: