Merge pull request #261 from takluyver/discovery

Prototype new kernel discovery machinery

Author: takluyver

docs/index.rst

@@ -23,6 +23,7 @@ with Jupyter kernels.
 
    kernels
    wrapperkernels
+   kernel_providers
 
 .. toctree::
    :maxdepth: 2

docs/kernel_providers.rst

@@ -0,0 +1,156 @@
+================
+Kernel providers
+================
+
+.. note::
+   This is a new interface under development, and may still change.
+   Not all Jupyter applications use this yet.
+   See :ref:`kernelspecs` for the established way of discovering kernel types.
+
+Creating a kernel provider
+==========================
+
+By writing a kernel provider, you can extend how Jupyter applications discover
+and start kernels. For example, you could find kernels in an environment system
+like conda, or kernels on remote systems which you can access.
+
+To write a kernel provider, subclass
+:class:`jupyter_client.discovery.KernelProviderBase`, giving your provider an ID
+and overriding two methods.
+
+.. class:: MyKernelProvider
+
+   .. attribute:: id
+
+      A short string identifying this provider. Cannot contain forward slash
+      (``/``).
+
+   .. method:: find_kernels()
+
+      Get the available kernel types this provider knows about.
+      Return an iterable of 2-tuples: (name, attributes).
+      *name* is a short string identifying the kernel type.
+      *attributes* is a dictionary with information to allow selecting a kernel.
+
+   .. method:: make_manager(name)
+
+      Prepare and return a :class:`~jupyter_client.KernelManager` instance
+      ready to start a new kernel instance of the type identified by *name*.
+      The input will be one of the names given by :meth:`find_kernels`.
+
+For example, imagine we want to tell Jupyter about kernels for a new language
+called *oblong*::
+
+    # oblong_provider.py
+    from jupyter_client.discover import KernelProviderBase
+    from jupyter_client import KernelManager
+    from shutil import which
+
+    class OblongKernelProvider(KernelProviderBase):
+        id = 'oblong'
+
+        def find_kernels(self):
+            if not which('oblong-kernel'):
+                return  # Check it's available
+
+            # Two variants - for a real kernel, these could be something like
+            # different conda environments.
+            yield 'standard', {
+                'display_name': 'Oblong (standard)',
+                'language': {'name': 'oblong'},
+                'argv': ['oblong-kernel'],
+            }
+            yield 'rounded', {
+                'display_name': 'Oblong (rounded)',
+                'language': {'name': 'oblong'},
+                'argv': ['oblong-kernel'],
+            }
+
+        def make_manager(self, name):
+            if name == 'standard':
+                return KernelManager(kernel_cmd=['oblong-kernel'],
+                                     extra_env={'ROUNDED': '0'})
+            elif name == 'rounded':
+                return KernelManager(kernel_cmd=['oblong-kernel'],
+                                     extra_env={'ROUNDED': '1'})
+            else:
+                raise ValueError("Unknown kernel %s" % name)
+
+You would then register this with an *entry point*. In your ``setup.py``, put
+something like this::
+
+    setup(...
+        entry_points = {
+        'jupyter_client.kernel_providers' : [
+            # The name before the '=' should match the id attribute
+            'oblong = oblong_provider:OblongKernelProvider',
+        ]
+    })
+
+Finding kernel types
+====================
+
+To find and start kernels in client code, use
+:class:`jupyter_client.discovery.KernelFinder`. This uses multiple kernel
+providers to find available kernels. Like a kernel provider, it has methods
+``find_kernels`` and ``make_manager``. The kernel names it works
+with have the provider ID as a prefix, e.g. ``oblong/rounded`` (from the example
+above).
+
+::
+
+    from jupyter_client.discovery import KernelFinder
+    kf = KernelFinder.from_entrypoints()
+
+    ## Find available kernel types
+    for name, attributes in kf.find_kernels():
+        print(name, ':', attributes['display_name'])
+    # oblong/standard : Oblong (standard)
+    # oblong/rounded : Oblong(rounded)
+    # ...
+
+    ## Start a kernel by name
+    manager = kf.make_manager('oblong/standard')
+    manager.start_kernel()
+
+.. module:: jupyter_client.discovery
+
+.. autoclass:: KernelFinder
+
+   .. automethod:: from_entrypoints
+
+   .. automethod:: find_kernels
+
+   .. automethod:: make_manager
+
+Kernel providers included in ``jupyter_client``
+===============================================
+
+``jupyter_client`` includes two kernel providers:
+
+.. autoclass:: KernelSpecProvider
+
+   .. seealso:: :ref:`kernelspecs`
+
+.. autoclass:: IPykernelProvider
+
+Glossary
+========
+
+Kernel instance
+  A running kernel, a process which can accept ZMQ connections from frontends.
+  Its state includes a namespace and an execution counter.
+
+Kernel type
+  The software to run a kernel instance, along with the context in which a
+  kernel starts. One kernel type allows starting multiple, initially similar
+  kernel instances. For instance, one kernel type may be associated with one
+  conda environment containing ``ipykernel``. The same kernel software in
+  another environment would be a different kernel type. Another software package
+  for a kernel, such as ``IRkernel``, would also be a different kernel type.
+
+Kernel provider
+  A Python class to discover kernel types and allow a client to start instances
+  of those kernel types. For instance, one kernel provider might find conda
+  environments containing ``ipykernel`` and allow starting kernel instances in
+  these environments.

jupyter_client/discovery.py

@@ -0,0 +1,131 @@
+from abc import ABCMeta, abstractmethod
+import entrypoints
+import logging
+import six
+
+from .kernelspec import KernelSpecManager
+from .manager import KernelManager
+
+log = logging.getLogger(__name__)
+
+class KernelProviderBase(six.with_metaclass(ABCMeta, object)):
+    id = None  # Should be a short string identifying the provider class.
+
+    @abstractmethod
+    def find_kernels(self):
+        """Return an iterator of (kernel_name, kernel_info_dict) tuples."""
+        pass
+
+    @abstractmethod
+    def make_manager(self, name):
+        """Make and return a KernelManager instance to start a specified kernel
+
+        name will be one of the kernel names produced by find_kernels()
+        """
+        pass
+
+class KernelSpecProvider(KernelProviderBase):
+    """Offers kernel types from installed kernelspec directories.
+    """
+    id = 'spec'
+
+    def __init__(self):
+        self.ksm = KernelSpecManager()
+
+    def find_kernels(self):
+        for name, resdir in self.ksm.find_kernel_specs().items():
+            spec = self.ksm._get_kernel_spec_by_name(name, resdir)
+            yield name, {
+                # TODO: get full language info
+                'language': {'name': spec.language},
+                'display_name': spec.display_name,
+                'argv': spec.argv,
+            }
+
+    def make_manager(self, name):
+        spec = self.ksm.get_kernel_spec(name)
+        return KernelManager(kernel_cmd=spec.argv, extra_env=spec.env)
+
+
+class IPykernelProvider(KernelProviderBase):
+    """Offers a kernel type using the Python interpreter it's running in.
+
+    This checks if ipykernel is importable first.
+    """
+    id = 'pyimport'
+
+    def _check_for_kernel(self):
+        try:
+            from ipykernel.kernelspec import RESOURCES, get_kernel_dict
+            from ipykernel.ipkernel import IPythonKernel
+        except ImportError:
+            return None
+        else:
+            return {
+                'spec': get_kernel_dict(),
+                'language_info': IPythonKernel.language_info,
+                'resources_dir': RESOURCES,
+            }
+
+    def find_kernels(self):
+        info = self._check_for_kernel()
+
+        if info:
+            yield 'kernel', {
+                'language': info['language_info'],
+                'display_name': info['spec']['display_name'],
+                'argv': info['spec']['argv'],
+            }
+
+    def make_manager(self, name):
+        info = self._check_for_kernel()
+        if info is None:
+            raise Exception("ipykernel is not importable")
+        return KernelManager(kernel_cmd=info['spec']['argv'])
+
+
+class KernelFinder(object):
+    """Manages a collection of kernel providers to find available kernel types
+
+    *providers* should be a list of kernel provider instances.
+    """
+    def __init__(self, providers):
+        self.providers = providers
+
+    @classmethod
+    def from_entrypoints(cls):
+        """Load all kernel providers advertised by entry points.
+
+        Kernel providers should use the "jupyter_client.kernel_providers"
+        entry point group.
+
+        Returns an instance of KernelFinder.
+        """
+        providers = []
+        for ep in entrypoints.get_group_all('jupyter_client.kernel_providers'):
+            try:
+                provider = ep.load()()  # Load and instantiate
+            except Exception:
+                log.error('Error loading kernel provider', exc_info=True)
+            else:
+                providers.append(provider)
+
+        return cls(providers)
+
+    def find_kernels(self):
+        """Iterate over available kernel types.
+
+        Yields 2-tuples of (prefixed_name, attributes)
+        """
+        for provider in self.providers:
+            for kid, attributes in provider.find_kernels():
+                id = provider.id + '/' + kid
+                yield id, attributes
+
+    def make_manager(self, name):
+        """Make a KernelManager instance for a given kernel type.
+        """
+        provider_id, kernel_id = name.split('/', 1)
+        for provider in self.providers:
+            if provider_id == provider.id:
+                return provider.make_manager(kernel_id)

jupyter_client/manager.py

@@ -22,7 +22,7 @@
 from ipython_genutils.importstring import import_item
 from .localinterfaces import is_local_ip, local_ips
 from traitlets import (
-    Any, Float, Instance, Unicode, List, Bool, Type, DottedObjectName
+    Any, Float, Instance, Unicode, List, Bool, Type, DottedObjectName, Dict
 )
 from jupyter_client import (
     launch_kernel,
@@ -87,23 +87,13 @@ def kernel_spec(self):
             self._kernel_spec = self.kernel_spec_manager.get_kernel_spec(self.kernel_name)
         return self._kernel_spec
 
-    kernel_cmd = List(Unicode(), config=True,
-        help="""DEPRECATED: Use kernel_name instead.
-
-        The Popen Command to launch the kernel.
-        Override this if you have a custom kernel.
-        If kernel_cmd is specified in a configuration file,
-        Jupyter does not pass any arguments to the kernel,
-        because it cannot make any assumptions about the
-        arguments that the kernel understands. In particular,
-        this means that the kernel does not receive the
-        option --debug if it given on the Jupyter command line.
-        """
+    kernel_cmd = List(Unicode(),
+        help="""The Popen Command to launch the kernel."""
     )
 
-    def _kernel_cmd_changed(self, name, old, new):
-        warnings.warn("Setting kernel_cmd is deprecated, use kernel_spec to "
-                      "start different kernels.")
+    extra_env = Dict(
+        help="""Extra environment variables to be set for the kernel."""
+    )
 
     @property
     def ipykernel(self):
@@ -254,6 +244,8 @@ def start_kernel(self, **kw):
             # If kernel_cmd has been set manually, don't refer to a kernel spec
             # Environment variables from kernel spec are added to os.environ
             env.update(self.kernel_spec.env or {})
+        elif self.extra_env:
+            env.update(self.extra_env)
 
         # launch the kernel subprocess
         self.log.debug("Starting kernel: %s", kernel_cmd)

jupyter_client/tests/test_discovery.py

@@ -0,0 +1,32 @@
+import sys
+
+from jupyter_client import KernelManager
+from jupyter_client import discovery
+
+def test_ipykernel_provider():
+    import ipykernel  # Fail clearly if ipykernel not installed
+    ikf = discovery.IPykernelProvider()
+
+    res = list(ikf.find_kernels())
+    assert len(res) == 1, res
+    id, info = res[0]
+    assert id == 'kernel'
+    assert info['argv'][0] == sys.executable
+
+class DummyKernelProvider(discovery.KernelProviderBase):
+    """A dummy kernel provider for testing KernelFinder"""
+    id = 'dummy'
+
+    def find_kernels(self):
+        yield 'sample', {'argv': ['dummy_kernel']}
+
+    def make_manager(self, name):
+        return KernelManager(kernel_cmd=['dummy_kernel'])
+
+def test_meta_kernel_finder():
+    kf = discovery.KernelFinder(providers=[DummyKernelProvider()])
+    assert list(kf.find_kernels()) == \
+        [('dummy/sample', {'argv': ['dummy_kernel']})]
+
+    manager = kf.make_manager('dummy/sample')
+    assert manager.kernel_cmd == ['dummy_kernel']

setup.py

@@ -82,6 +82,7 @@ def run(self):
         'jupyter_core',
         'pyzmq>=13',
         'python-dateutil>=2.1',
+        'entrypoints',
     ],
     extras_require   = {
         'test': ['ipykernel', 'ipython', 'mock', 'pytest'],
@@ -93,6 +94,10 @@ def run(self):
         'console_scripts': [
             'jupyter-kernelspec = jupyter_client.kernelspecapp:KernelSpecApp.launch_instance',
             'jupyter-run = jupyter_client.runapp:RunApp.launch_instance',
+        ],
+        'jupyter_client.kernel_providers' : [
+            'spec = jupyter_client.discovery:KernelSpecProvider',
+            'pyimport = jupyter_client.discovery:IPykernelProvider',
         ]
     },
 )