From d67c82a6e09c9316c58e52e4a1ce4badfc3bc71e Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Thu, 18 May 2017 17:47:56 +0100 Subject: [PATCH 01/48] Make a start on kernel discovery framework --- jupyter_client/discovery.py | 80 +++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 jupyter_client/discovery.py diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py new file mode 100644 index 000000000..7e640c32f --- /dev/null +++ b/jupyter_client/discovery.py @@ -0,0 +1,80 @@ +from .kernelspec import KernelSpecManager +from .manager import KernelManager + + +class KernelSpecFinder(object): + """Find kernels from installed kernelspec directories. + """ + id = 'spec' + + def __init__(self): + self.ksm = KernelSpecManager() + + def find_kernels(self): + for name, resdir in self.ksm.find_kernel_specs().items(): + spec = self.ksm._get_kernel_spec_by_name(name, resdir) + yield name, { + # TODO: get full language info + 'language': {'name': spec.language}, + 'display_name': spec.display_name, + 'argv': spec.argv, + } + + def make_manager(self, name): + spec = self.ksm.get_kernel_spec(name) + return KernelManager(kernel_cmd=spec.argv) # TODO: env + + +class IPykernelFinder(object): + """Find ipykernel on this Python version by trying to import it. + """ + id = 'pyimport' + + def _check_for_kernel(self): + try: + from ipykernel.kernelspec import RESOURCES, get_kernel_dict + from ipykernel.ipkernel import IPythonKernel + except ImportError: + return None + else: + return { + 'spec': get_kernel_dict(), + 'language_info': IPythonKernel.language_info, + 'resources_dir': RESOURCES, + } + + def find_kernels(self): + info = self._check_for_kernel() + + if info: + yield 'kernel', { + 'language': info['language_info'], + 'display_name': info['spec']['display_name'], + 'argv': info['spec']['argv'], + } + + def make_manager(self): + info = self._check_for_kernel() + if info is None: + raise Exception("ipykernel is not importable") + return KernelManager(kernel_cmd=info['spec']['argv']) + + +class MetaKernelFinder(object): + def __init__(self): + self.finders = [ + KernelSpecFinder(), + IPykernelFinder(), + ] + + def find_kernels(self): + for finder in self.finders: + for kid, attributes in finder.find_kernels(): + id = finder.id + '/' + kid + yield id, attributes + + def make_manager(self, id): + finder_id, kernel_id = id.split('/', 1) + for finder in self.finders: + if finder_id == finder.id: + return finder.make_manager(kernel_id) From 6406393ff0fcb66fc547f77f81541af1e507978b Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 14:31:37 +0100 Subject: [PATCH 02/48] Undeprecate KernelManager.kernel_cmd, add extra_env --- jupyter_client/discovery.py | 4 ++-- jupyter_client/manager.py | 24 ++++++++---------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 7e640c32f..737ee38b1 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -22,7 +22,7 @@ def find_kernels(self): def make_manager(self, name): spec = self.ksm.get_kernel_spec(name) - return KernelManager(kernel_cmd=spec.argv) # TODO: env + return KernelManager(kernel_cmd=spec.argv, extra_env=spec.env) class IPykernelFinder(object): @@ -53,7 +53,7 @@ def find_kernels(self): 'argv': info['spec']['argv'], } - def make_manager(self): + def make_manager(self, name): info = self._check_for_kernel() if info is None: raise Exception("ipykernel is not importable") diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index d50a5fbb8..4e0387762 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -22,7 +22,7 @@ from ipython_genutils.importstring import import_item from .localinterfaces import is_local_ip, local_ips from traitlets import ( - Any, Float, Instance, Unicode, List, Bool, Type, DottedObjectName + Any, Float, Instance, Unicode, List, Bool, Type, DottedObjectName, Dict ) from jupyter_client import ( launch_kernel, @@ -87,23 +87,13 @@ def kernel_spec(self): self._kernel_spec = self.kernel_spec_manager.get_kernel_spec(self.kernel_name) return self._kernel_spec - kernel_cmd = List(Unicode(), config=True, - help="""DEPRECATED: Use kernel_name instead. - - The Popen Command to launch the kernel. - Override this if you have a custom kernel. - If kernel_cmd is specified in a configuration file, - Jupyter does not pass any arguments to the kernel, - because it cannot make any assumptions about the - arguments that the kernel understands. In particular, - this means that the kernel does not receive the - option --debug if it given on the Jupyter command line. - """ + kernel_cmd = List(Unicode(), + help="""The Popen Command to launch the kernel.""" ) - def _kernel_cmd_changed(self, name, old, new): - warnings.warn("Setting kernel_cmd is deprecated, use kernel_spec to " - "start different kernels.") + extra_env = Dict( + help="""Extra environment variables to be set for the kernel.""" + ) @property def ipykernel(self): @@ -254,6 +244,8 @@ def start_kernel(self, **kw): # If kernel_cmd has been set manually, don't refer to a kernel spec # Environment variables from kernel spec are added to os.environ env.update(self.kernel_spec.env or {}) + elif self.extra_env: + env.update(self.extra_env) # launch the kernel subprocess self.log.debug("Starting kernel: %s", kernel_cmd) From 6ca3ec77ab675a0c1b31dbd3f67538589f1d63cc Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 26 Jul 2017 16:35:52 +0100 Subject: [PATCH 03/48] Use entry points to find kernel finders --- jupyter_client/discovery.py | 24 +++++++++++++++++++----- setup.py | 4 ++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 737ee38b1..d6ff84b68 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -1,6 +1,10 @@ +import entrypoints +import logging + from .kernelspec import KernelSpecManager from .manager import KernelManager +log = logging.getLogger(__name__) class KernelSpecFinder(object): """Find kernels from installed kernelspec directories. @@ -61,11 +65,21 @@ def make_manager(self, name): class MetaKernelFinder(object): - def __init__(self): - self.finders = [ - KernelSpecFinder(), - IPykernelFinder(), - ] + def __init__(self, finders): + self.finders = finders + + @classmethod + def from_entrypoints(cls): + finders = [] + for ep in entrypoints.get_group_all('jupyter_client.kernel_finders'): + try: + finder = ep.load()() # Load and instantiate + except Exception: + log.error('Error loading kernel finder', exc_info=True) + else: + finders.append(finder) + + return cls(finders) def find_kernels(self): for finder in self.finders: diff --git a/setup.py b/setup.py index 341af7fb2..a48d4c428 100644 --- a/setup.py +++ b/setup.py @@ -93,6 +93,10 @@ def run(self): 'console_scripts': [ 'jupyter-kernelspec = jupyter_client.kernelspecapp:KernelSpecApp.launch_instance', 'jupyter-run = jupyter_client.runapp:RunApp.launch_instance', + ], + 'jupyter_client.kernel_finders' : [ + 'spec = jupyter_client.discovery:KernelSpecFinder', + 'pyimport = jupyter_client.discovery:IPykernelFinder', ] }, ) From dddda322e93807b9b09af0d8bbf1967c4faa90b8 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 26 Jul 2017 16:56:34 +0100 Subject: [PATCH 04/48] Tests for kernel discovery machinery --- jupyter_client/discovery.py | 21 +++++++++++++++-- jupyter_client/tests/test_discovery.py | 32 ++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 jupyter_client/tests/test_discovery.py diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index d6ff84b68..f43cb48a4 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -1,3 +1,4 @@ +from abc import ABC, abstractmethod import entrypoints import logging @@ -6,7 +7,23 @@ log = logging.getLogger(__name__) -class KernelSpecFinder(object): +class KernelFinderBase(ABC): + id = None # Should be a short string identifying the finder class. + + @abstractmethod + def find_kernels(self): + """Return an iterator of (kernel_name, kernel_info_dict) tuples.""" + pass + + @abstractmethod + def make_manager(self, name): + """Make and return a KernelManager instance to start a specified kernel + + name will be one of the kernel names produced by find_kernels() + """ + pass + +class KernelSpecFinder(KernelFinderBase): """Find kernels from installed kernelspec directories. """ id = 'spec' @@ -29,7 +46,7 @@ def make_manager(self, name): return KernelManager(kernel_cmd=spec.argv, extra_env=spec.env) -class IPykernelFinder(object): +class IPykernelFinder(KernelFinderBase): """Find ipykernel on this Python version by trying to import it. """ id = 'pyimport' diff --git a/jupyter_client/tests/test_discovery.py b/jupyter_client/tests/test_discovery.py new file mode 100644 index 000000000..f6a462327 --- /dev/null +++ b/jupyter_client/tests/test_discovery.py @@ -0,0 +1,32 @@ +import sys + +from jupyter_client import KernelManager +from jupyter_client import discovery + +def test_ipykernel_finder(): + import ipykernel # Fail clearly if ipykernel not installed + ikf = discovery.IPykernelFinder() + + res = list(ikf.find_kernels()) + assert len(res) == 1, res + id, info = res[0] + assert id == 'kernel' + assert info['argv'][0] == sys.executable + +class DummyKernelFinder(discovery.KernelFinderBase): + """A dummy kernel finder for testing MetaKernelFinder""" + id = 'dummy' + + def find_kernels(self): + yield 'sample', {'argv': ['dummy_kernel']} + + def make_manager(self, name): + return KernelManager(kernel_cmd=['dummy_kernel']) + +def test_meta_kernel_finder(): + mkf = discovery.MetaKernelFinder(finders=[DummyKernelFinder()]) + assert list(mkf.find_kernels()) == \ + [('dummy/sample', {'argv': ['dummy_kernel']})] + + manager = mkf.make_manager('dummy/sample') + assert manager.kernel_cmd == ['dummy_kernel'] From 1509dacd4c526eb3976577ee14c557abc77c97ed Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 26 Jul 2017 17:47:10 +0100 Subject: [PATCH 05/48] Use older ABC definition style with metaclass --- jupyter_client/discovery.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index f43cb48a4..9fd4e6327 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -1,13 +1,14 @@ -from abc import ABC, abstractmethod +from abc import ABCMeta, abstractmethod import entrypoints import logging +import six from .kernelspec import KernelSpecManager from .manager import KernelManager log = logging.getLogger(__name__) -class KernelFinderBase(ABC): +class KernelFinderBase(six.with_metaclass(ABCMeta, object)): id = None # Should be a short string identifying the finder class. @abstractmethod From 38ccbdc0751a35d46abca80ae64a106d492841cd Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 9 Oct 2017 15:21:28 +0100 Subject: [PATCH 06/48] Rename kernel finders -> kernel providers MetaKernelFinder -> KernelFinder --- jupyter_client/discovery.py | 55 ++++++++++++++++---------- jupyter_client/tests/test_discovery.py | 14 +++---- setup.py | 2 +- 3 files changed, 43 insertions(+), 28 deletions(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 9fd4e6327..53ccab11b 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -8,8 +8,8 @@ log = logging.getLogger(__name__) -class KernelFinderBase(six.with_metaclass(ABCMeta, object)): - id = None # Should be a short string identifying the finder class. +class KernelProviderBase(six.with_metaclass(ABCMeta, object)): + id = None # Should be a short string identifying the provider class. @abstractmethod def find_kernels(self): @@ -24,7 +24,7 @@ def make_manager(self, name): """ pass -class KernelSpecFinder(KernelFinderBase): +class KernelSpecProvider(KernelProviderBase): """Find kernels from installed kernelspec directories. """ id = 'spec' @@ -47,7 +47,7 @@ def make_manager(self, name): return KernelManager(kernel_cmd=spec.argv, extra_env=spec.env) -class IPykernelFinder(KernelFinderBase): +class IPykernelProvider(KernelProviderBase): """Find ipykernel on this Python version by trying to import it. """ id = 'pyimport' @@ -82,31 +82,46 @@ def make_manager(self, name): return KernelManager(kernel_cmd=info['spec']['argv']) -class MetaKernelFinder(object): - def __init__(self, finders): - self.finders = finders +class KernelFinder(object): + """Manages a collection of kernel providers to find available kernels + """ + def __init__(self, providers): + self.providers = providers @classmethod def from_entrypoints(cls): - finders = [] - for ep in entrypoints.get_group_all('jupyter_client.kernel_finders'): + """Load all kernel providers advertised by entry points. + + Kernel providers should use the "jupyter_client.kernel_providers" + entry point group. + + Returns an instance of KernelFinder. + """ + providers = [] + for ep in entrypoints.get_group_all('jupyter_client.kernel_providers'): try: - finder = ep.load()() # Load and instantiate + provider = ep.load()() # Load and instantiate except Exception: - log.error('Error loading kernel finder', exc_info=True) + log.error('Error loading kernel provider', exc_info=True) else: - finders.append(finder) + providers.append(provider) - return cls(finders) + return cls(providers) def find_kernels(self): - for finder in self.finders: - for kid, attributes in finder.find_kernels(): - id = finder.id + '/' + kid + """Iterate over available kernels. + + Yields 2-tuples of (id_str, attributes) + """ + for provider in self.providers: + for kid, attributes in provider.find_kernels(): + id = provider.id + '/' + kid yield id, attributes def make_manager(self, id): - finder_id, kernel_id = id.split('/', 1) - for finder in self.finders: - if finder_id == finder.id: - return finder.make_manager(kernel_id) + """Make a KernelManager instance for a given kernel ID. + """ + provider_id, kernel_id = id.split('/', 1) + for provider in self.providers: + if provider_id == provider.id: + return provider.make_manager(kernel_id) diff --git a/jupyter_client/tests/test_discovery.py b/jupyter_client/tests/test_discovery.py index f6a462327..9d7833ba3 100644 --- a/jupyter_client/tests/test_discovery.py +++ b/jupyter_client/tests/test_discovery.py @@ -3,9 +3,9 @@ from jupyter_client import KernelManager from jupyter_client import discovery -def test_ipykernel_finder(): +def test_ipykernel_provider(): import ipykernel # Fail clearly if ipykernel not installed - ikf = discovery.IPykernelFinder() + ikf = discovery.IPykernelProvider() res = list(ikf.find_kernels()) assert len(res) == 1, res @@ -13,8 +13,8 @@ def test_ipykernel_finder(): assert id == 'kernel' assert info['argv'][0] == sys.executable -class DummyKernelFinder(discovery.KernelFinderBase): - """A dummy kernel finder for testing MetaKernelFinder""" +class DummyKernelProvider(discovery.KernelProviderBase): + """A dummy kernel provider for testing KernelFinder""" id = 'dummy' def find_kernels(self): @@ -24,9 +24,9 @@ def make_manager(self, name): return KernelManager(kernel_cmd=['dummy_kernel']) def test_meta_kernel_finder(): - mkf = discovery.MetaKernelFinder(finders=[DummyKernelFinder()]) - assert list(mkf.find_kernels()) == \ + kf = discovery.KernelFinder(providers=[DummyKernelProvider()]) + assert list(kf.find_kernels()) == \ [('dummy/sample', {'argv': ['dummy_kernel']})] - manager = mkf.make_manager('dummy/sample') + manager = kf.make_manager('dummy/sample') assert manager.kernel_cmd == ['dummy_kernel'] diff --git a/setup.py b/setup.py index a48d4c428..2399a8139 100644 --- a/setup.py +++ b/setup.py @@ -94,7 +94,7 @@ def run(self): 'jupyter-kernelspec = jupyter_client.kernelspecapp:KernelSpecApp.launch_instance', 'jupyter-run = jupyter_client.runapp:RunApp.launch_instance', ], - 'jupyter_client.kernel_finders' : [ + 'jupyter_client.kernel_providers' : [ 'spec = jupyter_client.discovery:KernelSpecFinder', 'pyimport = jupyter_client.discovery:IPykernelFinder', ] From 3c09a5732d569be86fd92f06b3deeab6413c4e65 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 9 Oct 2017 15:26:52 +0100 Subject: [PATCH 07/48] Missed a rename --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2399a8139..099ddc90a 100644 --- a/setup.py +++ b/setup.py @@ -95,8 +95,8 @@ def run(self): 'jupyter-run = jupyter_client.runapp:RunApp.launch_instance', ], 'jupyter_client.kernel_providers' : [ - 'spec = jupyter_client.discovery:KernelSpecFinder', - 'pyimport = jupyter_client.discovery:IPykernelFinder', + 'spec = jupyter_client.discovery:KernelSpecProvider', + 'pyimport = jupyter_client.discovery:IPykernelProvider', ] }, ) From e92e5c194adde5188039269255a0f82c2c45627d Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 9 Oct 2017 15:41:43 +0100 Subject: [PATCH 08/48] Add dependency on entrypoints --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 099ddc90a..f042f00b3 100644 --- a/setup.py +++ b/setup.py @@ -82,6 +82,7 @@ def run(self): 'jupyter_core', 'pyzmq>=13', 'python-dateutil>=2.1', + 'entrypoints', ], extras_require = { 'test': ['ipykernel', 'ipython', 'mock', 'pytest'], From aad40cb6ebb785f8ce51ee70b9872d62aef09e32 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 10 Oct 2017 15:17:58 +0100 Subject: [PATCH 09/48] Document new kernel providers system --- docs/index.rst | 1 + docs/kernel_providers.rst | 146 ++++++++++++++++++++++++++++++++++++ jupyter_client/discovery.py | 18 +++-- 3 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 docs/kernel_providers.rst diff --git a/docs/index.rst b/docs/index.rst index a0b8855cc..41e218ccc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,7 @@ with Jupyter kernels. kernels wrapperkernels + kernel_providers .. toctree:: :maxdepth: 2 diff --git a/docs/kernel_providers.rst b/docs/kernel_providers.rst new file mode 100644 index 000000000..8d51ec7f4 --- /dev/null +++ b/docs/kernel_providers.rst @@ -0,0 +1,146 @@ +================ +Kernel providers +================ + +.. note:: + This is a new interface under development. Not all Jupyter applications + use this yet. See :ref:`kernelspecs` for the established way of discovering + kernel types. + +By writing a kernel provider, you can extend how Jupyter applications discover +and start kernels. To do so, subclass +:class:`jupyter_client.discovery.KernelProviderBase`, giving your provider an ID +and overriding two methods. + +.. class:: MyKernelProvider + + .. attribute:: id + + A short string identifying this provider. Cannot contain forward slash + (``/``). + + .. method:: find_kernels() + + Get the available kernel types this provider knows about. + Return an iterable of 2-tuples: (name, attributes). + *name* is a short string identifying the kernel type. + *attributes* is a dictionary with information to allow selecting a kernel. + + .. method:: make_manager(name) + + Prepare and return a :class:`~jupyter_client.KernelManager` instance + ready to start a new kernel instance of the type identified by *name*. + The input will be one of the names given by :meth:`find_kernels`. + +For example, imagine we want to tell Jupyter about kernels for a new language +called *oblong*:: + + # oblong_provider.py + from jupyter_client.discover import KernelProviderBase + from jupyter_client import KernelManager + from shutil import which + + class OblongKernelProvider(KernelProviderBase): + id = 'oblong' + + def find_kernels(self): + if not which('oblong-kernel'): + return # Check it's available + + # Two variants - for a real kernel, these could be different + # environments + yield 'standard', { + 'display_name': 'Oblong (standard)', + 'language': {'name': 'oblong'}, + 'argv': ['oblong-kernel'], + } + yield 'rounded', { + 'display_name': 'Oblong (rounded)', + 'language': {'name': 'oblong'}, + 'argv': ['oblong-kernel'], + } + + def make_manager(self, name): + if name == 'standard': + return KernelManager(kernel_cmd=['oblong-kernel'], + extra_env={'ROUNDED': '0'}) + elif name == 'rounded': + return KernelManager(kernel_cmd=['oblong-kernel'], + extra_env={'ROUNDED': '1'}) + else: + raise ValueError("Unknown kernel %s" % name) + +You would then register this with an *entry point*. In your ``setup.py``, put +something like this:: + + setup(... + entry_points = { + 'jupyter_client.kernel_providers' : [ + # The name before the '=' should match the id attribute + 'oblong = oblong_provider:OblongKernelProvider', + ] + }) + +To find and start kernels in client code, use +:class:`jupyter_client.discovery.KernelFinder`. This has a similar API to kernel +providers, but it wraps a set of kernel providers. The kernel names it works +with have the provider ID as a prefix, e.g. ``oblong/rounded`` (from the example +above). + +:: + + from jupyter_client.discovery import KernelFinder + kf = KernelFinder.from_entrypoints() + + ## Find available kernel types + for name, attributes in kf.find_kernels(): + print(name, ':', attributes['display_name']) + # oblong/standard : Oblong (standard) + # oblong/rounded : Oblong(rounded) + # ... + + ## Start a kernel by name + manager = kf.make_manager('oblong/standard') + manager.start_kernel() + +.. module:: jupyter_client.discovery + +.. autoclass:: KernelFinder + + .. automethod:: from_entrypoints + + .. automethod:: find_kernels + + .. automethod:: make_manager + +Included kernel providers +========================= + +``jupyter_client`` includes two kernel providers: + +.. autoclass:: KernelSpecProvider + + .. seealso:: :ref:`kernelspecs` + +.. autoclass:: IPykernelProvider + +Glossary +======== + +Kernel instance + A running kernel, a process which can accept ZMQ connections from frontends. + Its state includes a namespace and an execution counter. + +Kernel type + Allows starting multiple, initially similar kernel instances. The kernel type + entails the combination of software to run the kernel, and the context in + which it starts. For instance, one kernel type may be associated with one + conda environment containing ``ipykernel``. The same kernel software in + another environment would be a different kernel type. Another software package + for a kernel, such as ``IRkernel``, would also be a different kernel type. + +Kernel provider + A Python class to discover kernel types and allow a client to start instances + of those kernel types. For instance, one kernel provider might find conda + environments containing ``ipykernel`` and allow starting kernel instances in + these environments. diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 53ccab11b..6f6b52f0d 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -25,7 +25,7 @@ def make_manager(self, name): pass class KernelSpecProvider(KernelProviderBase): - """Find kernels from installed kernelspec directories. + """Offers kernel types from installed kernelspec directories. """ id = 'spec' @@ -48,7 +48,9 @@ def make_manager(self, name): class IPykernelProvider(KernelProviderBase): - """Find ipykernel on this Python version by trying to import it. + """Offers a kernel type using the Python interpreter it's running in. + + This checks if ipykernel is importable first. """ id = 'pyimport' @@ -83,7 +85,9 @@ def make_manager(self, name): class KernelFinder(object): - """Manages a collection of kernel providers to find available kernels + """Manages a collection of kernel providers to find available kernel types + + *providers* should be a list of kernel provider instances. """ def __init__(self, providers): self.providers = providers @@ -109,17 +113,17 @@ def from_entrypoints(cls): return cls(providers) def find_kernels(self): - """Iterate over available kernels. + """Iterate over available kernel types. - Yields 2-tuples of (id_str, attributes) + Yields 2-tuples of (prefixed_name, attributes) """ for provider in self.providers: for kid, attributes in provider.find_kernels(): id = provider.id + '/' + kid yield id, attributes - def make_manager(self, id): - """Make a KernelManager instance for a given kernel ID. + def make_manager(self, name): + """Make a KernelManager instance for a given kernel type. """ provider_id, kernel_id = id.split('/', 1) for provider in self.providers: From c09b8aced5912901b4e96e11d9d4eeb3346b4fd7 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 10 Oct 2017 15:25:41 +0100 Subject: [PATCH 10/48] Break it up a bit with a subheading --- docs/kernel_providers.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/kernel_providers.rst b/docs/kernel_providers.rst index 8d51ec7f4..c5a62cc3d 100644 --- a/docs/kernel_providers.rst +++ b/docs/kernel_providers.rst @@ -81,6 +81,9 @@ something like this:: ] }) +Finding kernel types +==================== + To find and start kernels in client code, use :class:`jupyter_client.discovery.KernelFinder`. This has a similar API to kernel providers, but it wraps a set of kernel providers. The kernel names it works From cc8176b088fdf7a5ea955fec980c6b90e8bd21f7 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 10 Oct 2017 18:06:43 +0100 Subject: [PATCH 11/48] Update doc with Carol's suggestions --- docs/kernel_providers.rst | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/docs/kernel_providers.rst b/docs/kernel_providers.rst index c5a62cc3d..65cbd9c8f 100644 --- a/docs/kernel_providers.rst +++ b/docs/kernel_providers.rst @@ -3,12 +3,18 @@ Kernel providers ================ .. note:: - This is a new interface under development. Not all Jupyter applications - use this yet. See :ref:`kernelspecs` for the established way of discovering - kernel types. + This is a new interface under development, and may still change. + Not all Jupyter applications use this yet. + See :ref:`kernelspecs` for the established way of discovering kernel types. + +Creating a kernel provider +========================== By writing a kernel provider, you can extend how Jupyter applications discover -and start kernels. To do so, subclass +and start kernels. For example, you could find kernels in an environment system +like conda, or kernels on remote systems which you can access. + +To write a kernel provider, subclass :class:`jupyter_client.discovery.KernelProviderBase`, giving your provider an ID and overriding two methods. @@ -47,8 +53,8 @@ called *oblong*:: if not which('oblong-kernel'): return # Check it's available - # Two variants - for a real kernel, these could be different - # environments + # Two variants - for a real kernel, these could be something like + # different conda environments. yield 'standard', { 'display_name': 'Oblong (standard)', 'language': {'name': 'oblong'}, @@ -85,8 +91,9 @@ Finding kernel types ==================== To find and start kernels in client code, use -:class:`jupyter_client.discovery.KernelFinder`. This has a similar API to kernel -providers, but it wraps a set of kernel providers. The kernel names it works +:class:`jupyter_client.discovery.KernelFinder`. This uses multiple kernel +providers to find available kernels. Like a kernel provider, it has methods +``find_kernels`` and ``make_manager``. The kernel names it works with have the provider ID as a prefix, e.g. ``oblong/rounded`` (from the example above). @@ -116,8 +123,8 @@ above). .. automethod:: make_manager -Included kernel providers -========================= +Kernel providers included in ``jupyter_client`` +=============================================== ``jupyter_client`` includes two kernel providers: @@ -135,9 +142,9 @@ Kernel instance Its state includes a namespace and an execution counter. Kernel type - Allows starting multiple, initially similar kernel instances. The kernel type - entails the combination of software to run the kernel, and the context in - which it starts. For instance, one kernel type may be associated with one + The software to run a kernel instance, along with the context in which a + kernel starts. One kernel type allows starting multiple, initially similar + kernel instances. For instance, one kernel type may be associated with one conda environment containing ``ipykernel``. The same kernel software in another environment would be a different kernel type. Another software package for a kernel, such as ``IRkernel``, would also be a different kernel type. From 1f74c5f40f9948b1978451ceb473beb319cd7649 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 10 Oct 2017 18:07:53 +0100 Subject: [PATCH 12/48] Fix variable name --- jupyter_client/discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 6f6b52f0d..2bfe92b2a 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -125,7 +125,7 @@ def find_kernels(self): def make_manager(self, name): """Make a KernelManager instance for a given kernel type. """ - provider_id, kernel_id = id.split('/', 1) + provider_id, kernel_id = name.split('/', 1) for provider in self.providers: if provider_id == provider.id: return provider.make_manager(kernel_id) From 16608fc7835cba0bbd9e62feac31481dcd517c64 Mon Sep 17 00:00:00 2001 From: didier amyot Date: Wed, 18 Oct 2017 20:29:25 -0400 Subject: [PATCH 13/48] Fix typo in documentation. --- docs/kernel_providers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kernel_providers.rst b/docs/kernel_providers.rst index 65cbd9c8f..2e1b7e295 100644 --- a/docs/kernel_providers.rst +++ b/docs/kernel_providers.rst @@ -42,7 +42,7 @@ For example, imagine we want to tell Jupyter about kernels for a new language called *oblong*:: # oblong_provider.py - from jupyter_client.discover import KernelProviderBase + from jupyter_client.discovery import KernelProviderBase from jupyter_client import KernelManager from shutil import which From 936dfe0584441ababc8e6d86740f4791f7739a19 Mon Sep 17 00:00:00 2001 From: frelon Date: Wed, 1 Nov 2017 12:56:55 +0100 Subject: [PATCH 14/48] Updated URL for Jupyter Kernels The old URL points to a "This page has moved"-page --- docs/kernels.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kernels.rst b/docs/kernels.rst index 3319dda31..2fe1500aa 100644 --- a/docs/kernels.rst +++ b/docs/kernels.rst @@ -6,7 +6,7 @@ Making kernels for Jupyter A 'kernel' is a program that runs and introspects the user's code. IPython includes a kernel for Python code, and people have written kernels for -`several other languages `_. +`several other languages `_. When Jupyter starts a kernel, it passes it a connection file. This specifies how to set up communications with the frontend. From aca5f7084014ec69d51f5141a4fd1bdfb1aa3a3b Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 10 Nov 2017 14:43:08 +0100 Subject: [PATCH 15/48] tornado 5 support - use IOLoop.current over IOLoop.instance - drop removed `loop` arg from PeriodicCallback - deprecate now-unused IOLoopKernelRestarter.loop --- jupyter_client/ioloop/manager.py | 20 ++++---------------- jupyter_client/ioloop/restarter.py | 27 +++++++++------------------ jupyter_client/session.py | 4 ++-- 3 files changed, 15 insertions(+), 36 deletions(-) diff --git a/jupyter_client/ioloop/manager.py b/jupyter_client/ioloop/manager.py index 511a73f55..cc285291b 100644 --- a/jupyter_client/ioloop/manager.py +++ b/jupyter_client/ioloop/manager.py @@ -1,15 +1,7 @@ """A kernel manager with a tornado IOLoop""" -#----------------------------------------------------------------------------- -# Copyright (c) The Jupyter Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. from __future__ import absolute_import @@ -24,10 +16,6 @@ from jupyter_client.manager import KernelManager from .restarter import IOLoopKernelRestarter -#----------------------------------------------------------------------------- -# Code -#----------------------------------------------------------------------------- - def as_zmqstream(f): def wrapped(self, *args, **kwargs): @@ -37,9 +25,9 @@ def wrapped(self, *args, **kwargs): class IOLoopKernelManager(KernelManager): - loop = Instance('zmq.eventloop.ioloop.IOLoop') + loop = Instance('tornado.ioloop.IOLoop') def _loop_default(self): - return ioloop.IOLoop.instance() + return ioloop.IOLoop.current() restarter_class = Type( default_value=IOLoopKernelRestarter, diff --git a/jupyter_client/ioloop/restarter.py b/jupyter_client/ioloop/restarter.py index 6f531744c..69079eecf 100644 --- a/jupyter_client/ioloop/restarter.py +++ b/jupyter_client/ioloop/restarter.py @@ -4,37 +4,28 @@ restarts the kernel if it dies. """ -#----------------------------------------------------------------------------- -# Copyright (c) The Jupyter Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. from __future__ import absolute_import +import warnings from zmq.eventloop import ioloop - from jupyter_client.restarter import KernelRestarter from traitlets import ( Instance, ) -#----------------------------------------------------------------------------- -# Code -#----------------------------------------------------------------------------- - class IOLoopKernelRestarter(KernelRestarter): """Monitor and autorestart a kernel.""" - loop = Instance('zmq.eventloop.ioloop.IOLoop') + loop = Instance('tornado.ioloop.IOLoop') def _loop_default(self): - return ioloop.IOLoop.instance() + warnings.warn("IOLoopKernelRestarter.loop is deprecated in jupyter-client 5.2", + DeprecationWarning, stacklevel=4, + ) + return ioloop.IOLoop.current() _pcallback = None @@ -42,7 +33,7 @@ def start(self): """Start the polling of the kernel.""" if self._pcallback is None: self._pcallback = ioloop.PeriodicCallback( - self.poll, 1000*self.time_to_dead, self.loop + self.poll, 1000*self.time_to_dead, ) self._pcallback.start() diff --git a/jupyter_client/session.py b/jupyter_client/session.py index af60ac259..33b1c0b4a 100644 --- a/jupyter_client/session.py +++ b/jupyter_client/session.py @@ -191,9 +191,9 @@ def _context_default(self): session = Instance('jupyter_client.session.Session', allow_none=True) - loop = Instance('zmq.eventloop.ioloop.IOLoop') + loop = Instance('tornado.ioloop.IOLoop') def _loop_default(self): - return IOLoop.instance() + return IOLoop.current() def __init__(self, **kwargs): super(SessionFactory, self).__init__(**kwargs) From 172d6cdea80bf189a894171fdd39cc6031ae562d Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 21 Sep 2017 15:04:12 +0200 Subject: [PATCH 16/48] Configure interrupt mode via spec. - interrupt_mode="signal" is the default and current behaviour - With interrupt_mode="message", instead of a signal, a `interrupt_request` message on the control port will be sent --- jupyter_client/kernelspec.py | 18 ++++++++++++------ jupyter_client/manager.py | 19 +++++++++++++------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 3465ac7a4..d2248cc58 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -13,7 +13,9 @@ pjoin = os.path.join from ipython_genutils.py3compat import PY3 -from traitlets import HasTraits, List, Unicode, Dict, Set, Bool, Type +from traitlets import ( + HasTraits, List, Unicode, Dict, Set, Bool, Type, CaselessStrEnum +) from traitlets.config import LoggingConfigurable from jupyter_core.paths import jupyter_data_dir, jupyter_path, SYSTEM_JUPYTER_PATH @@ -28,6 +30,9 @@ class KernelSpec(HasTraits): language = Unicode() env = Dict() resource_dir = Unicode() + interrupt_mode = CaselessStrEnum( + ['message', 'signal'], default_value='signal' + ) metadata = Dict() @classmethod @@ -46,6 +51,7 @@ def to_dict(self): env=self.env, display_name=self.display_name, language=self.language, + interrupt_mode=self.interrupt_mode, metadata=self.metadata, ) @@ -227,7 +233,7 @@ def get_all_specs(self): def remove_kernel_spec(self, name): """Remove a kernel spec directory by name. - + Returns the path that was deleted. """ save_native = self.ensure_native_kernel @@ -263,7 +269,7 @@ def install_kernel_spec(self, source_dir, kernel_name=None, user=False, If ``user`` is False, it will attempt to install into the systemwide kernel registry. If the process does not have appropriate permissions, an :exc:`OSError` will be raised. - + If ``prefix`` is given, the kernelspec will be installed to PREFIX/share/jupyter/kernels/KERNEL_NAME. This can be sys.prefix for installation inside virtual or conda envs. @@ -284,16 +290,16 @@ def install_kernel_spec(self, source_dir, kernel_name=None, user=False, DeprecationWarning, stacklevel=2, ) - + destination = self._get_destination_dir(kernel_name, user=user, prefix=prefix) self.log.debug('Installing kernelspec in %s', destination) - + kernel_dir = os.path.dirname(destination) if kernel_dir not in self.kernel_dirs: self.log.warning("Installing to %s, which is not in %s. The kernelspec may not be found.", kernel_dir, self.kernel_dirs, ) - + if os.path.isdir(destination): self.log.info('Removing existing kernelspec in %s', destination) shutil.rmtree(destination) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 4e0387762..2bcc1629a 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -246,7 +246,7 @@ def start_kernel(self, **kw): env.update(self.kernel_spec.env or {}) elif self.extra_env: env.update(self.extra_env) - + # launch the kernel subprocess self.log.debug("Starting kernel: %s", kernel_cmd) self.kernel = self._launch_kernel(kernel_cmd, env=env, @@ -403,11 +403,18 @@ def interrupt_kernel(self): platforms. """ if self.has_kernel: - if sys.platform == 'win32': - from .win_interrupt import send_interrupt - send_interrupt(self.kernel.win32_interrupt_event) - else: - self.signal_kernel(signal.SIGINT) + interrupt_mode = self.kernel_spec.interrupt_mode + if interrupt_mode == 'signal': + if sys.platform == 'win32': + from .win_interrupt import send_interrupt + send_interrupt(self.kernel.win32_interrupt_event) + else: + self.signal_kernel(signal.SIGINT) + + elif interrupt_mode == 'message': + msg = self.session.msg("interrupt_request", content={}) + self._connect_control_socket() + self.session.send(self._control_socket, msg) else: raise RuntimeError("Cannot interrupt kernel. No kernel is running!") From f0e33ba7532ab50ffd54fbc0912edc61815eba03 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 30 Oct 2017 16:30:12 +0100 Subject: [PATCH 17/48] Update docs. --- docs/kernels.rst | 7 +++++++ docs/messaging.rst | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/docs/kernels.rst b/docs/kernels.rst index 2fe1500aa..76fa67699 100644 --- a/docs/kernels.rst +++ b/docs/kernels.rst @@ -132,6 +132,13 @@ JSON serialised dictionary containing the following keys and values: is found, a kernel with a matching `language` will be used. This allows a notebook written on any Python or Julia kernel to be properly associated with the user's Python or Julia kernel, even if they aren't listed under the same name as the author's. +- **interrupt_mode** (optional): May be either ``signal`` or ``message`` and + specifies how a client is supposed to interrupt cell execution on this kernel, + either by sending an interrupt ``signal`` via the operating system's + signalling facilities (e.g. `SIGTERM` on POSIX systems), or by sending an + ``interrupt_request`` message on the control channel (see + :ref:`msging_interrupt`). If this is not specified + the client will default to ``signal`` mode. - **env** (optional): A dictionary of environment variables to set for the kernel. These will be added to the current environment variables before the kernel is started. diff --git a/docs/messaging.rst b/docs/messaging.rst index 776dda681..ec8efd99f 100644 --- a/docs/messaging.rst +++ b/docs/messaging.rst @@ -959,6 +959,27 @@ Message type: ``shutdown_reply``:: socket, they simply send a forceful process termination signal, since a dead process is unlikely to respond in any useful way to messages. +.. _msging_interrupt: + +Kernel interrupt +---------------- + +In case a kernel can not catch operating system interrupt signals (e.g. the used +runtime handles signals and does not allow a user program to define a callback), +a kernel can choose to be notified using a message instead. For this to work, +the kernels kernelspec must set `interrupt_mode` to ``message``. An interruption +will then result in the following message on the `control` channel: + +Message type: ``interrupt_request``:: + + content = {} + +Message type: ``interrupt_reply``:: + + content = {} + +.. versionadded:: 5.3 + Messages on the IOPub (PUB/SUB) channel ======================================= From 21b95699dcb5917a8cf87c8aae3bd67b9e281f3c Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 6 Nov 2017 11:59:10 +0100 Subject: [PATCH 18/48] Bump protocol version. --- docs/messaging.rst | 2 +- jupyter_client/_version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/messaging.rst b/docs/messaging.rst index ec8efd99f..7c533a7de 100644 --- a/docs/messaging.rst +++ b/docs/messaging.rst @@ -21,7 +21,7 @@ Versioning The Jupyter message specification is versioned independently of the packages that use it. -The current version of the specification is 5.2. +The current version of the specification is 5.3. .. note:: *New in* and *Changed in* messages in this document refer to versions of the diff --git a/jupyter_client/_version.py b/jupyter_client/_version.py index 90dd2e93e..7f96345ae 100644 --- a/jupyter_client/_version.py +++ b/jupyter_client/_version.py @@ -1,5 +1,5 @@ version_info = (5, 1, 0) __version__ = '.'.join(map(str, version_info)) -protocol_version_info = (5, 2) +protocol_version_info = (5, 3) protocol_version = "%i.%i" % protocol_version_info From 6674afae21cce681c1fae6a37879d40c181cc91c Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 13 Nov 2017 14:31:20 +0100 Subject: [PATCH 19/48] disable pyzmq zero-copy optimizations during session tests --- jupyter_client/tests/test_session.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/jupyter_client/tests/test_session.py b/jupyter_client/tests/test_session.py index 43819a898..e80274367 100644 --- a/jupyter_client/tests/test_session.py +++ b/jupyter_client/tests/test_session.py @@ -8,6 +8,10 @@ import sys import uuid from datetime import datetime +try: + from unittest import mock +except ImportError: + import mock import pytest @@ -34,6 +38,14 @@ def setUp(self): self.session = ss.Session() +@pytest.fixture +def no_copy_threshold(): + """Disable zero-copy optimizations in pyzmq >= 17""" + with mock.patch.object(zmq, 'COPY_THRESHOLD', 1): + yield + + +@pytest.mark.usefixtures('no_copy_threshold') class TestSession(SessionTestCase): def test_msg(self): From e2772bd54c864b805b1cac36b3141fe27b1ba726 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 13 Nov 2017 15:11:37 +0100 Subject: [PATCH 20/48] Fix signal name. --- docs/kernels.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kernels.rst b/docs/kernels.rst index 76fa67699..5308c603f 100644 --- a/docs/kernels.rst +++ b/docs/kernels.rst @@ -135,7 +135,7 @@ JSON serialised dictionary containing the following keys and values: - **interrupt_mode** (optional): May be either ``signal`` or ``message`` and specifies how a client is supposed to interrupt cell execution on this kernel, either by sending an interrupt ``signal`` via the operating system's - signalling facilities (e.g. `SIGTERM` on POSIX systems), or by sending an + signalling facilities (e.g. `SIGINT` on POSIX systems), or by sending an ``interrupt_request`` message on the control channel (see :ref:`msging_interrupt`). If this is not specified the client will default to ``signal`` mode. From 948d653e86a3923f23ffd92117f104c8bbc234c3 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 22 Nov 2017 13:01:11 +0100 Subject: [PATCH 21/48] extend special handling of sys.executable to pythonX[.Y] this should allow ipykernel's wheel-installed specs to specify `python3` or `python2` and prevent python2 kernels from launching with sys.executable if the Python version is 3. --- jupyter_client/manager.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 2bcc1629a..52bf8b781 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -11,11 +11,6 @@ import signal import sys import time -import warnings -try: - from queue import Empty # Py 3 -except ImportError: - from Queue import Empty # Py 2 import zmq @@ -29,7 +24,6 @@ kernelspec, ) from .connect import ConnectionFileMixin -from .session import Session from .managerabc import ( KernelManagerABC ) @@ -164,8 +158,10 @@ def format_kernel_cmd(self, extra_arguments=None): else: cmd = self.kernel_spec.argv + extra_arguments - if cmd and cmd[0] == 'python': - # executable is 'python', use sys.executable. + if cmd and cmd[0] in {'python', + 'python%i' % sys.version_info[0], + 'python%i.%i' % sys.version_info[:2]}: + # executable is 'python' or 'python3', use sys.executable. # These will typically be the same, # but if the current process is in an env # and has been launched by abspath without From 250178fe53dcf5c20098e29ea94951caa3aa371e Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 10 Feb 2017 16:57:22 +0000 Subject: [PATCH 22/48] Add 'jupyter kernel' command A simple lead in to the 'kernel nanny' work, this adds a command so you can do: jupyter kernel --kernel python --- jupyter_client/kernelapp.py | 66 +++++++++++++++++++++++++++++++++++++ scripts/jupyter-kernel | 5 +++ setup.py | 1 + 3 files changed, 72 insertions(+) create mode 100644 jupyter_client/kernelapp.py create mode 100755 scripts/jupyter-kernel diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py new file mode 100644 index 000000000..4c1c99e3c --- /dev/null +++ b/jupyter_client/kernelapp.py @@ -0,0 +1,66 @@ +import os +import signal +import uuid + +from jupyter_core.application import JupyterApp +from tornado.ioloop import IOLoop +from traitlets import Unicode + +from . import __version__ +from .kernelspec import KernelSpecManager +from .manager import KernelManager + +class KernelApp(JupyterApp): + version = __version__ + description = "Run a kernel locally" + + classes = [KernelManager, KernelSpecManager] + + aliases = { + 'kernel': 'KernelApp.kernel_name', + 'ip': 'KernelManager.ip', + } + + kernel_name = Unicode( + help = 'The name of a kernel to start' + ).tag(config=True) + + def initialize(self, argv=None): + super(KernelApp, self).initialize(argv) + self.km = KernelManager(kernel_name=self.kernel_name, + config=self.config) + cf_basename = 'kernel-%s.json' % uuid.uuid4() + self.km.connection_file = os.path.join(self.runtime_dir, cf_basename) + self.loop = IOLoop.current() + + def setup_signals(self): + if os.name == 'nt': + return + + def shutdown_handler(signo, frame): + self.loop.add_callback_from_signal(self.shutdown, signo) + for sig in [signal.SIGTERM, signal.SIGINT]: + signal.signal(sig, shutdown_handler) + + def shutdown(self, signo): + self.log.info('Shutting down on signal %d' % signo) + self.km.shutdown_kernel() + self.loop.stop() + + def log_connection_info(self): + cf = self.km.connection_file + self.log.info('Connection file: %s', cf) + self.log.info("To connect a client: --existing %s", os.path.basename(cf)) + + def start(self): + self.log.info('Starting kernel %r', self.kernel_name) + try: + self.km.start_kernel() + self.log_connection_info() + self.setup_signals() + self.loop.start() + finally: + self.km.cleanup() + + +main = KernelApp.launch_instance diff --git a/scripts/jupyter-kernel b/scripts/jupyter-kernel new file mode 100755 index 000000000..31144d405 --- /dev/null +++ b/scripts/jupyter-kernel @@ -0,0 +1,5 @@ +#!/usr/bin/env python +from jupyter_client.kernelapp import main + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index f042f00b3..022cbc56e 100644 --- a/setup.py +++ b/setup.py @@ -94,6 +94,7 @@ def run(self): 'console_scripts': [ 'jupyter-kernelspec = jupyter_client.kernelspecapp:KernelSpecApp.launch_instance', 'jupyter-run = jupyter_client.runapp:RunApp.launch_instance', + 'jupyter-kernel = jupyter_client.kernelapp:main', ], 'jupyter_client.kernel_providers' : [ 'spec = jupyter_client.discovery:KernelSpecProvider', From 9359b338c90f8e259ac4b307d99ce20ca3b2cbf7 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 10:23:18 +0000 Subject: [PATCH 23/48] Use native kernel by default --- jupyter_client/kernelapp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py index 4c1c99e3c..071a0f3ed 100644 --- a/jupyter_client/kernelapp.py +++ b/jupyter_client/kernelapp.py @@ -7,7 +7,7 @@ from traitlets import Unicode from . import __version__ -from .kernelspec import KernelSpecManager +from .kernelspec import KernelSpecManager, NATIVE_KERNEL_NAME from .manager import KernelManager class KernelApp(JupyterApp): @@ -21,7 +21,7 @@ class KernelApp(JupyterApp): 'ip': 'KernelManager.ip', } - kernel_name = Unicode( + kernel_name = Unicode(NATIVE_KERNEL_NAME, help = 'The name of a kernel to start' ).tag(config=True) From ae03ddde10c215a8df1efe4a29d5bfa91b1efdfa Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 10:32:40 +0000 Subject: [PATCH 24/48] More description --- jupyter_client/kernelapp.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py index 071a0f3ed..799d85ee4 100644 --- a/jupyter_client/kernelapp.py +++ b/jupyter_client/kernelapp.py @@ -2,7 +2,7 @@ import signal import uuid -from jupyter_core.application import JupyterApp +from jupyter_core.application import JupyterApp, base_flags from tornado.ioloop import IOLoop from traitlets import Unicode @@ -11,8 +11,10 @@ from .manager import KernelManager class KernelApp(JupyterApp): + """Launch a kernel by name in a local subprocess. + """ version = __version__ - description = "Run a kernel locally" + description = "Run a kernel locally in a subprocess" classes = [KernelManager, KernelSpecManager] @@ -20,9 +22,10 @@ class KernelApp(JupyterApp): 'kernel': 'KernelApp.kernel_name', 'ip': 'KernelManager.ip', } + flags = {'debug': base_flags['debug']} kernel_name = Unicode(NATIVE_KERNEL_NAME, - help = 'The name of a kernel to start' + help = 'The name of a kernel type to start' ).tag(config=True) def initialize(self, argv=None): @@ -34,6 +37,7 @@ def initialize(self, argv=None): self.loop = IOLoop.current() def setup_signals(self): + """Shutdown on SIGTERM or SIGINT (Ctrl-C)""" if os.name == 'nt': return From 7e6d16711c6f16782a497dc6dbf76911c334f46e Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 11:22:38 +0000 Subject: [PATCH 25/48] Add test of 'jupyter kernel' --- jupyter_client/kernelapp.py | 11 +++++ jupyter_client/tests/test_kernelapp.py | 57 ++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 jupyter_client/tests/test_kernelapp.py diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py index 799d85ee4..a2ab17812 100644 --- a/jupyter_client/kernelapp.py +++ b/jupyter_client/kernelapp.py @@ -35,6 +35,7 @@ def initialize(self, argv=None): cf_basename = 'kernel-%s.json' % uuid.uuid4() self.km.connection_file = os.path.join(self.runtime_dir, cf_basename) self.loop = IOLoop.current() + self.loop.add_callback(self._record_started) def setup_signals(self): """Shutdown on SIGTERM or SIGINT (Ctrl-C)""" @@ -56,6 +57,16 @@ def log_connection_info(self): self.log.info('Connection file: %s', cf) self.log.info("To connect a client: --existing %s", os.path.basename(cf)) + def _record_started(self): + """For tests, create a file to indicate that we've started + + Do not rely on this except in our own tests! + """ + fn = os.environ.get('JUPYTER_CLIENT_TEST_RECORD_STARTUP_PRIVATE') + if fn is not None: + with open(fn, 'wb'): + pass + def start(self): self.log.info('Starting kernel %r', self.kernel_name) try: diff --git a/jupyter_client/tests/test_kernelapp.py b/jupyter_client/tests/test_kernelapp.py new file mode 100644 index 000000000..b41a02bc6 --- /dev/null +++ b/jupyter_client/tests/test_kernelapp.py @@ -0,0 +1,57 @@ +from __future__ import division + +import os +import shutil +from subprocess import Popen, PIPE +import sys +from tempfile import mkdtemp +import time + +def _launch(extra_env): + env = os.environ.copy() + env.update(extra_env) + return Popen([sys.executable, '-c', + 'from jupyter_client.kernelapp import main; main()'], + env=env, stderr=PIPE) + +WAIT_TIME = 10 +POLL_FREQ = 10 + +def test_kernelapp_lifecycle(): + # Check that 'jupyter kernel' starts and terminates OK. + runtime_dir = mkdtemp() + startup_dir = mkdtemp() + started = os.path.join(startup_dir, 'started') + try: + p = _launch({'JUPYTER_RUNTIME_DIR': runtime_dir, + 'JUPYTER_CLIENT_TEST_RECORD_STARTUP_PRIVATE': started, + }) + # Wait for start + for _ in range(WAIT_TIME * POLL_FREQ): + if os.path.isfile(started): + break + time.sleep(1 / POLL_FREQ) + else: + raise AssertionError("No started file created in {} seconds" + .format(WAIT_TIME)) + + # Connection file should be there by now + files = os.listdir(runtime_dir) + assert len(files) == 1 + cf = files[0] + assert cf.startswith('kernel') + assert cf.endswith('.json') + + # Read the first three lines from stderr. This will hang if there are + # fewer lines to read; I don't see any way to avoid that without lots + # of extra complexity. + b = b''.join(p.stderr.readline() for _ in range(2)).decode('utf-8', 'replace') + assert cf in b + + # Send SIGTERM to shut down + p.terminate() + p.wait(timeout=10) + finally: + shutil.rmtree(runtime_dir) + shutil.rmtree(startup_dir) + From 28f908f0da34ed0e0c85f58016334c434c18bb5f Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 11:32:35 +0000 Subject: [PATCH 26/48] Workaround lack of timeout on Py2 --- jupyter_client/tests/test_kernelapp.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/jupyter_client/tests/test_kernelapp.py b/jupyter_client/tests/test_kernelapp.py index b41a02bc6..2533472d4 100644 --- a/jupyter_client/tests/test_kernelapp.py +++ b/jupyter_client/tests/test_kernelapp.py @@ -7,16 +7,28 @@ from tempfile import mkdtemp import time +PY3 = sys.version_info[0] >= 3 + def _launch(extra_env): env = os.environ.copy() env.update(extra_env) return Popen([sys.executable, '-c', 'from jupyter_client.kernelapp import main; main()'], - env=env, stderr=PIPE) + env=env, stderr=(PIPE if PY3 else None)) WAIT_TIME = 10 POLL_FREQ = 10 +def hacky_wait(p): + """Python 2 subprocess doesn't have timeouts :-(""" + for _ in range(WAIT_TIME * POLL_FREQ): + if p.poll() is not None: + return p.returncode + time.sleep(1 / POLL_FREQ) + else: + raise AssertionError("Process didn't exit in {} seconds" + .format(WAIT_TIME)) + def test_kernelapp_lifecycle(): # Check that 'jupyter kernel' starts and terminates OK. runtime_dir = mkdtemp() @@ -42,15 +54,13 @@ def test_kernelapp_lifecycle(): assert cf.startswith('kernel') assert cf.endswith('.json') - # Read the first three lines from stderr. This will hang if there are - # fewer lines to read; I don't see any way to avoid that without lots - # of extra complexity. - b = b''.join(p.stderr.readline() for _ in range(2)).decode('utf-8', 'replace') - assert cf in b - # Send SIGTERM to shut down p.terminate() - p.wait(timeout=10) + if PY3: + _, stderr = p.communicate(timeout=WAIT_TIME) + assert cf in stderr.decode('utf-8', 'replace') + else: + hacky_wait(p) finally: shutil.rmtree(runtime_dir) shutil.rmtree(startup_dir) From aa8b184c9c8134cae731c4652a87a704b6ee9f65 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 11:34:19 +0000 Subject: [PATCH 27/48] Restrict to older pytest on Python 3.3 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 022cbc56e..1230f2142 100644 --- a/setup.py +++ b/setup.py @@ -86,6 +86,7 @@ def run(self): ], extras_require = { 'test': ['ipykernel', 'ipython', 'mock', 'pytest'], + 'test:python_version == "3.3"': ['pytest<3.3.0'], }, cmdclass = { 'bdist_egg': bdist_egg if 'bdist_egg' in sys.argv else bdist_egg_disabled, From 5291f940c8cac341ed96c6b2dd73bbdd11db1df5 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 11:36:43 +0000 Subject: [PATCH 28/48] Another go at fixing pytest dependency on Python 3.3 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1230f2142..233f83a0e 100644 --- a/setup.py +++ b/setup.py @@ -85,8 +85,9 @@ def run(self): 'entrypoints', ], extras_require = { - 'test': ['ipykernel', 'ipython', 'mock', 'pytest'], + 'test': ['ipykernel', 'ipython', 'mock'], 'test:python_version == "3.3"': ['pytest<3.3.0'], + 'test:python_version >= "3.4" or python_version == "2.7"': ['pytest'], }, cmdclass = { 'bdist_egg': bdist_egg if 'bdist_egg' in sys.argv else bdist_egg_disabled, From ed051077267f02bf630c32cce20406a52240474c Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 15 Dec 2017 11:57:57 +0000 Subject: [PATCH 29/48] Tolerate invalid kernel specs in get_all_specs() --- jupyter_client/kernelspec.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index d2248cc58..132d8f9c2 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -226,10 +226,17 @@ def get_all_specs(self): } """ d = self.find_kernel_specs() - return {kname: { - "resource_dir": d[kname], - "spec": self._get_kernel_spec_by_name(kname, d[kname]).to_dict() - } for kname in d} + res = {} + for kname, resource_dir in d.items(): + try: + spec = self._get_kernel_spec_by_name(kname, resource_dir) + res[kname] = { + "resource_dir": resource_dir, + "spec": spec.to_dict() + } + except Exception: + self.log.warning("Error loading kernelspec %r", kname, exc_info=True) + return res def remove_kernel_spec(self, name): """Remove a kernel spec directory by name. From dd4a2d652de2ea898ae6213c148b59c60579afba Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 15 Dec 2017 12:23:26 +0000 Subject: [PATCH 30/48] Improve performance of get_kernel_spec --- jupyter_client/kernelspec.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index d2248cc58..1a815970b 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -3,6 +3,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +import errno import io import json import os @@ -199,15 +200,39 @@ def _get_kernel_spec_by_name(self, kernel_name, resource_dir): return self.kernel_spec_class.from_resource_dir(resource_dir) + def _find_spec_directory(self, kernel_name): + """Find the resource directory of a named kernel spec""" + for kernel_dir in self.kernel_dirs: + try: + files = os.listdir(kernel_dir) + except OSError as e: + if e.errno in (errno.ENOTDIR, errno.ENOENT): + continue + raise + for f in files: + path = pjoin(kernel_dir, f) + if f.lower() == kernel_name and _is_kernel_dir(path): + return path + + if kernel_name == NATIVE_KERNEL_NAME: + try: + from ipykernel.kernelspec import RESOURCES + except ImportError: + pass + else: + return RESOURCES + def get_kernel_spec(self, kernel_name): """Returns a :class:`KernelSpec` instance for the given kernel_name. Raises :exc:`NoSuchKernel` if the given kernel name is not found. """ - d = self.find_kernel_specs() - try: - resource_dir = d[kernel_name.lower()] - except KeyError: + if not _is_valid_kernel_name(kernel_name): + self.log.warning("Kernelspec name %r is invalid: %s", kernel_name, + _kernel_name_description) + + resource_dir = self._find_spec_directory(kernel_name.lower()) + if resource_dir is None: raise NoSuchKernel(kernel_name) return self._get_kernel_spec_by_name(kernel_name, resource_dir) From 22092fa65ee0fd007bf59bfc7998b7d57743f0d2 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 15 Dec 2017 14:21:41 +0100 Subject: [PATCH 31/48] kill process group when killing kernel if killpg is available this should cleanup process trees (e.g. multiprocessing subprocesses) and make EADDRINUSE less likely during restart. --- jupyter_client/manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 52bf8b781..f488d5c42 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -372,7 +372,10 @@ def _kill_kernel(self): # Signal the kernel to terminate (sends SIGKILL on Unix and calls # TerminateProcess() on Win32). try: - self.kernel.kill() + if hasattr(signal, 'SIGKILL'): + self.signal_kernel(signal.SIGKILL) + else: + self.kernel.kill() except OSError as e: # In Windows, we will get an Access Denied error if the process # has already terminated. Ignore it. From 5f076b7321abd41f0ecb20080bba02c3e63f2b66 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 15 Dec 2017 14:09:39 +0000 Subject: [PATCH 32/48] Start writing release notes for 5.2 --- docs/changelog.rst | 36 ++++++++++++++++++++++++++++++++++++ docs/conf.py | 3 +++ docs/environment.yml | 2 ++ 3 files changed, 41 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 35e21b5c6..1f44e346f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,42 @@ Changes in Jupyter Client ========================= +5.2 +=== + +`5.2 on GitHub `__ + +- Define Jupyter protocol version 5.3: + + - Kernels can now opt to be interrupted by a message sent on the control channel + instead of a system signal. See :ref:`kernelspecs` and :ref:`msging_interrupt` + (:ghpull:`294`). + +- New ``jupyter kernel`` command to launch an installed kernel by name + (:ghpull:`240`). +- Kernelspecs where the command starts with e.g. ``python3`` or + ``python3.6``—matching the version ``jupyter_client`` is running on—are now + launched with the same Python executable as the launching process (:ghpull:`306`). + This extends the special handling of ``python`` added in 5.0. +- Command line arguments specified by a kernelspec can now include + ``{resource_dir}``, which will be substituted with the kernelspec resource + directory path when the kernel is launched (:ghpull:`289`). +- Kernelspecs now have an optional ``metadata`` field to hold arbitrary metadata + about kernels—see :ref:`kernelspecs` (:ghpull:`274`). +- Make the ``KernelRestarter`` class used by a ``KernelManager`` configurable + (:ghpull:`290`). +- If a kernel dies soon after starting, reassign random ports before restarting + it, in case one of the previously chosen ports has been bound by another + process (:ghpull:`279`). +- Check for non-contiguous buffers before trying to send them through ZMQ + (:ghpull:`258`). +- Compatibility with upcoming Tornado version 5.0 (:ghpull:`304`). +- Simplify setup code by always using setuptools (:ghpull:`284`). +- Soften warnings when setting the sticky bit on runtime files fails + (:ghpull:`286`). +- Various corrections and improvements to documentation. + + 5.1 === diff --git a/docs/conf.py b/docs/conf.py index 849d7a56e..c3de08efd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,6 +34,7 @@ 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.napoleon', + 'sphinxcontrib_github_alt', ] # Add any paths that contain templates here, relative to this directory. @@ -55,6 +56,8 @@ copyright = '2015, Jupyter Development Team' author = 'Jupyter Development Team' +github_project_url = "https://github.com/jupyter/jupyter_client" + # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. diff --git a/docs/environment.yml b/docs/environment.yml index 3690c73b7..459e7ab3b 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -8,3 +8,5 @@ dependencies: - jupyter_core - sphinx>=1.3.6 - sphinx_rtd_theme +- pip: + - sphinxcontrib_github_alt From 6689764905a8916e1c553e943e82c9d6870f2ae6 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 15 Dec 2017 14:25:36 +0000 Subject: [PATCH 33/48] Add PR #314 to changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1f44e346f..a78c79986 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -28,6 +28,7 @@ Changes in Jupyter Client about kernels—see :ref:`kernelspecs` (:ghpull:`274`). - Make the ``KernelRestarter`` class used by a ``KernelManager`` configurable (:ghpull:`290`). +- When killing a kernel on Unix, kill its process group (:ghpull:`314`). - If a kernel dies soon after starting, reassign random ports before restarting it, in case one of the previously chosen ports has been bound by another process (:ghpull:`279`). From 6251cf68e58ce89c3751ec7e356e86ee350f3652 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Sat, 16 Dec 2017 21:14:52 +0000 Subject: [PATCH 34/48] Add PRs #310 and #311 to changelog --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a78c79986..23cc0c953 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -32,6 +32,11 @@ Changes in Jupyter Client - If a kernel dies soon after starting, reassign random ports before restarting it, in case one of the previously chosen ports has been bound by another process (:ghpull:`279`). +- Avoid unnecessary filesystem operations when finding a kernelspec with + :meth:`.KernelSpecManager.get_kernel_spec` (:ghpull:`311`). +- :meth:`.KernelSpecManager.get_all_specs` will no longer raise an exception on + encountering an invalid ``kernel.json`` file. It will raise a warning and + continue (:ghpull:`310`). - Check for non-contiguous buffers before trying to send them through ZMQ (:ghpull:`258`). - Compatibility with upcoming Tornado version 5.0 (:ghpull:`304`). From a8b474512ff10120f46bea370c09bd6526bb14fb Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 19 Dec 2017 15:52:50 +0100 Subject: [PATCH 35/48] require tornado --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 233f83a0e..5de9cfd18 100644 --- a/setup.py +++ b/setup.py @@ -83,6 +83,7 @@ def run(self): 'pyzmq>=13', 'python-dateutil>=2.1', 'entrypoints', + 'tornado>=4.1', ], extras_require = { 'test': ['ipykernel', 'ipython', 'mock'], From 072a08727927ed0ac5c0b2cca0ccaa364109c060 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 30 Dec 2017 16:54:12 +0100 Subject: [PATCH 36/48] Parenthesize conditional requirement in setup.py Du to a likely bug in wheel, the conditional dependency on pytest ends up being unconditional. Seem like adding parenthesis fix that (as a work around). See https://github.com/pypa/setuptools/issues/1242 Closes #324 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 23cc0c953..d560cbfda 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changes in Jupyter Client ========================= +5.2.1 +===== + +- Add parenthesis to conditional pytest requirement to work around a bug in the + ``wheel`` package, that generate a ``.whl`` which otherwise always depends on + ``pytest`` see :ghissue:`324` and :ghpull:`325` + 5.2 === diff --git a/setup.py b/setup.py index 5de9cfd18..c184b40fa 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ def run(self): extras_require = { 'test': ['ipykernel', 'ipython', 'mock'], 'test:python_version == "3.3"': ['pytest<3.3.0'], - 'test:python_version >= "3.4" or python_version == "2.7"': ['pytest'], + 'test:(python_version >= "3.4" or python_version == "2.7")': ['pytest'], }, cmdclass = { 'bdist_egg': bdist_egg if 'bdist_egg' in sys.argv else bdist_egg_disabled, From e426a64ee0e0409201fc6817b0c01f3f5b2d3602 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 3 Jan 2018 14:25:48 +0100 Subject: [PATCH 37/48] Exclude build docs from sdist. This shrinks the sdist from 2MB to ~250KB... just realized that after uploading 5.2.1 took way too long. Apparently 5.2.0 alsho shipped built docs. --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 42edd273d..994648d70 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,7 @@ include README.md # Documentation graft docs exclude docs/\#* +exclude docs/_* # Examples graft examples From 41c5954a8257634b2d7f8905ed8c874946de606c Mon Sep 17 00:00:00 2001 From: stonebig Date: Sat, 6 Jan 2018 16:29:04 +0100 Subject: [PATCH 38/48] more complete error message to help inqiure on this https://github.com/jupyter/jupyter_client/issues/329 --- jupyter_client/manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index f488d5c42..21f6ca925 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -219,9 +219,10 @@ def start_kernel(self, **kw): """ if self.transport == 'tcp' and not is_local_ip(self.ip): raise RuntimeError("Can only launch a kernel on a local interface. " + "This one is not: %s." "Make sure that the '*_address' attributes are " "configured properly. " - "Currently valid addresses are: %s" % local_ips() + "Currently valid addresses are: %s" % (self.ip, local_ips()) ) # write connection file / get default ports From c658076da5c76455429e58b7d0a7510d3fa8d5be Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 6 Jan 2018 19:03:11 +0100 Subject: [PATCH 39/48] Tell Travis not to test the push from MrMeeseeks Use the ability to exclude branches as describe there: - https://docs.travis-ci.com/user/customizing-the-build/#Safelisting-or-blocklisting-branches Relatively easy as MrMeeseeks push a known branch format. This of course cannot be tested until merged and backported, and another backport triggered. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0a3a96915..faec1b44c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,3 +17,6 @@ after_success: matrix: allow_failures: - python: nightly +branches: + except: + - /^auto-backport-of-pr-[0-9]+$/ From cd735300df4df8d82a1abd0a27e8e56beaeb03af Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 16 Jan 2018 09:51:20 -0800 Subject: [PATCH 40/48] handle classes having been torn down in atexit we could probably avoid this if we registered/unregistered atexit callbacks for instances instead of registering it once for classes at import time --- jupyter_client/channels.py | 5 ++++- jupyter_client/threaded.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/jupyter_client/channels.py b/jupyter_client/channels.py index dd9906723..64e565189 100644 --- a/jupyter_client/channels.py +++ b/jupyter_client/channels.py @@ -80,7 +80,10 @@ def __init__(self, context=None, session=None, address=None): @staticmethod @atexit.register def _notice_exit(): - HBChannel._exiting = True + # Class definitions can be torn down during interpreter shutdown. + # We only need to set _exiting flag if this hasn't happened. + if HBChannel is not None: + HBChannel._exiting = True def _create_socket(self): if self.socket is not None: diff --git a/jupyter_client/threaded.py b/jupyter_client/threaded.py index f437aa58b..fda3a084c 100644 --- a/jupyter_client/threaded.py +++ b/jupyter_client/threaded.py @@ -151,7 +151,10 @@ def __init__(self, loop): @staticmethod @atexit.register def _notice_exit(): - IOLoopThread._exiting = True + # Class definitions can be torn down during interpreter shutdown. + # We only need to set _exiting flag if this hasn't happened. + if IOLoopThread is not None: + IOLoopThread._exiting = True def run(self): """Run my loop, ignoring EINTR events in the poller""" From 7d6812feb69230a98f39881369aadde9ad654a7c Mon Sep 17 00:00:00 2001 From: Sergey Kukhtichev Date: Mon, 29 Jan 2018 07:44:00 +0100 Subject: [PATCH 41/48] Add logic for detecting not responding kernels --- jupyter_client/restarter.py | 54 ++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/jupyter_client/restarter.py b/jupyter_client/restarter.py index 83d4356d0..5017dcda8 100644 --- a/jupyter_client/restarter.py +++ b/jupyter_client/restarter.py @@ -13,6 +13,11 @@ from traitlets import ( Instance, Float, Dict, Bool, Integer, ) +import time +try: + from queue import Empty # Python 3 +except ImportError: + from Queue import Empty # Python 2 class KernelRestarter(LoggingConfigurable): @@ -31,6 +36,10 @@ class KernelRestarter(LoggingConfigurable): help="""Kernel heartbeat interval in seconds.""" ) + startup_time = Float(20.0, config=True, + help="""Waiting time for kernel_info reply during initial startup""" + ) + restart_limit = Integer(5, config=True, help="""The number of consecutive autorestarts before the kernel is presumed dead.""" ) @@ -38,9 +47,16 @@ class KernelRestarter(LoggingConfigurable): random_ports_until_alive = Bool(True, config=True, help="""Whether to choose new random ports when restarting before the kernel is alive.""" ) + + kernel_monitor_enabled = Bool(True, config=True, + help="""Whether to restart kernel with new ports if response is not received within startup_time timeout""" + ) _restarting = Bool(False) _restart_count = Integer(0) _initial_startup = Bool(True) + _kernel_info_requested = Bool(False) + _kernel_info_timeout = Float(0) + kernel_client = None callbacks = Dict() def _callbacks_default(self): @@ -79,23 +95,22 @@ def remove_callback(self, f, event='restart'): except ValueError: pass - def _fire_callbacks(self, event): + def _fire_callbacks(self, event, **kwargs): """fire our callbacks for a particular event""" for callback in self.callbacks[event]: try: - callback() + callback(**kwargs) except Exception as e: self.log.error("KernelRestarter: %s callback %r failed", event, callback, exc_info=True) def poll(self): if self.debug: self.log.debug('Polling kernel...') - if not self.kernel_manager.is_alive(): + if not self.kernel_manager.is_alive() or self.is_kernel_response_timedout(): if self._restarting: self._restart_count += 1 else: self._restart_count = 1 - if self._restart_count >= self.restart_limit: self.log.warning("KernelRestarter: restart failed") self._fire_callbacks('dead') @@ -109,12 +124,37 @@ def poll(self): self.restart_limit, 'new' if newports else 'keep' ) - self._fire_callbacks('restart') self.kernel_manager.restart_kernel(now=True, newports=newports) + self._fire_callbacks('restart', newports=newports) self._restarting = True else: - if self._initial_startup: - self._initial_startup = False if self._restarting: self.log.debug("KernelRestarter: restart apparently succeeded") self._restarting = False + + def is_kernel_response_timedout(self): + if self.kernel_monitor_enabled: + if not self._kernel_info_requested: + self.kernel_client = self.kernel_manager.client() + self._kernel_info_timeout = time.time() + self.startup_time + self.log.info("KernelRestarter: Requesting kernel info") + self.kernel_client.kernel_info() + self._kernel_info_requested = True + if time.time() > self._kernel_info_timeout: + self.log.info("KernelRestarter: Kernel Info reply timed out") + self._kernel_info_requested = False + self._restarting = True + return True + try: + msg = self.kernel_client.shell_channel.get_msg(block=True, timeout=0) + except Empty: + self.log.info("KernelRestarter: No message received") + pass + else: + if msg['msg_type'] == 'kernel_info_reply': + self.kernel_client._handle_kernel_info_reply(msg) + self.log.info("KernelRestarter: Kernel info reply received") + self.kernel_monitor_enabled = False + if self._initial_startup: + self._initial_startup = False + return False \ No newline at end of file From 4d81491f4c8ad11a4f434b20b746be124d0ea45f Mon Sep 17 00:00:00 2001 From: Sergey Kukhtichev Date: Mon, 29 Jan 2018 15:11:12 +0100 Subject: [PATCH 42/48] changed log level for KernelRestarter messages --- jupyter_client/restarter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jupyter_client/restarter.py b/jupyter_client/restarter.py index 5017dcda8..133ebff5e 100644 --- a/jupyter_client/restarter.py +++ b/jupyter_client/restarter.py @@ -137,18 +137,18 @@ def is_kernel_response_timedout(self): if not self._kernel_info_requested: self.kernel_client = self.kernel_manager.client() self._kernel_info_timeout = time.time() + self.startup_time - self.log.info("KernelRestarter: Requesting kernel info") + self.log.debug("KernelRestarter: Requesting kernel info") self.kernel_client.kernel_info() self._kernel_info_requested = True if time.time() > self._kernel_info_timeout: - self.log.info("KernelRestarter: Kernel Info reply timed out") + self.log.warning("KernelRestarter: Kernel Info reply timed out. Restarting kernel") self._kernel_info_requested = False self._restarting = True return True try: msg = self.kernel_client.shell_channel.get_msg(block=True, timeout=0) except Empty: - self.log.info("KernelRestarter: No message received") + self.log.debug("KernelRestarter: No message received") pass else: if msg['msg_type'] == 'kernel_info_reply': From 5a8e5366d6a2093f3c2f88d45dd4678f69b4c54b Mon Sep 17 00:00:00 2001 From: Sergey Kukhtichev Date: Mon, 29 Jan 2018 15:49:36 +0100 Subject: [PATCH 43/48] enable kernel monitor only if the startup_time > 0 --- jupyter_client/restarter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_client/restarter.py b/jupyter_client/restarter.py index 133ebff5e..783e6b811 100644 --- a/jupyter_client/restarter.py +++ b/jupyter_client/restarter.py @@ -133,7 +133,7 @@ def poll(self): self._restarting = False def is_kernel_response_timedout(self): - if self.kernel_monitor_enabled: + if self.kernel_monitor_enabled and self.startup_time > 0: if not self._kernel_info_requested: self.kernel_client = self.kernel_manager.client() self._kernel_info_timeout = time.time() + self.startup_time From f655041ac87fc45ce29f359a79d402d1faedb8ad Mon Sep 17 00:00:00 2001 From: Sergey Kukhtichev Date: Mon, 29 Jan 2018 16:03:39 +0100 Subject: [PATCH 44/48] Added comments --- jupyter_client/restarter.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/jupyter_client/restarter.py b/jupyter_client/restarter.py index 783e6b811..d76675c98 100644 --- a/jupyter_client/restarter.py +++ b/jupyter_client/restarter.py @@ -133,6 +133,12 @@ def poll(self): self._restarting = False def is_kernel_response_timedout(self): + """ + Sends kernel_info request and checks a response. If the response is not received within kernel info timeout + then returns True, and sets marks kernel as _restarting. If the response received or kernel startup timeout + is not expired then returns False. + :return: bool + """ if self.kernel_monitor_enabled and self.startup_time > 0: if not self._kernel_info_requested: self.kernel_client = self.kernel_manager.client() From 2618f081fee1535dcddde9256413dbd160e2fe6e Mon Sep 17 00:00:00 2001 From: Sergey Kukhtichev Date: Tue, 30 Jan 2018 15:00:13 +0100 Subject: [PATCH 45/48] kernel_monitor_enabled configuration duplicates startup_time configuration. Made kernel_monitor_enabled non configurable --- jupyter_client/restarter.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/jupyter_client/restarter.py b/jupyter_client/restarter.py index d76675c98..8035d7cc5 100644 --- a/jupyter_client/restarter.py +++ b/jupyter_client/restarter.py @@ -37,7 +37,8 @@ class KernelRestarter(LoggingConfigurable): ) startup_time = Float(20.0, config=True, - help="""Waiting time for kernel_info reply during initial startup""" + help="""Waiting time for kernel_info reply during initial startup. + 0 indicates that kernel_info reply check disabled.""" ) restart_limit = Integer(5, config=True, @@ -48,14 +49,12 @@ class KernelRestarter(LoggingConfigurable): help="""Whether to choose new random ports when restarting before the kernel is alive.""" ) - kernel_monitor_enabled = Bool(True, config=True, - help="""Whether to restart kernel with new ports if response is not received within startup_time timeout""" - ) _restarting = Bool(False) _restart_count = Integer(0) _initial_startup = Bool(True) _kernel_info_requested = Bool(False) _kernel_info_timeout = Float(0) + _kernel_monitor_enabled = Bool(True) kernel_client = None callbacks = Dict() @@ -139,7 +138,7 @@ def is_kernel_response_timedout(self): is not expired then returns False. :return: bool """ - if self.kernel_monitor_enabled and self.startup_time > 0: + if self._kernel_monitor_enabled and self.startup_time > 0: if not self._kernel_info_requested: self.kernel_client = self.kernel_manager.client() self._kernel_info_timeout = time.time() + self.startup_time @@ -160,7 +159,7 @@ def is_kernel_response_timedout(self): if msg['msg_type'] == 'kernel_info_reply': self.kernel_client._handle_kernel_info_reply(msg) self.log.info("KernelRestarter: Kernel info reply received") - self.kernel_monitor_enabled = False + self._kernel_monitor_enabled = False if self._initial_startup: self._initial_startup = False return False \ No newline at end of file From fe2f5465d0462d12e506202bd7df4afaf17da4a9 Mon Sep 17 00:00:00 2001 From: Sergey Kukhtichev Date: Tue, 30 Jan 2018 16:09:11 +0100 Subject: [PATCH 46/48] Check whether callback function supports kwargs --- jupyter_client/restarter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jupyter_client/restarter.py b/jupyter_client/restarter.py index 8035d7cc5..d7fbaa35e 100644 --- a/jupyter_client/restarter.py +++ b/jupyter_client/restarter.py @@ -99,6 +99,9 @@ def _fire_callbacks(self, event, **kwargs): for callback in self.callbacks[event]: try: callback(**kwargs) + except TypeError: + self.log.warning("KernelRestarter: arguments are not supported by callback. Arguments will not be set") + callback() except Exception as e: self.log.error("KernelRestarter: %s callback %r failed", event, callback, exc_info=True) From 8d8a5bc175f14fd73d953fa07909b309c3792c50 Mon Sep 17 00:00:00 2001 From: Sergey Kukhtichev Date: Tue, 30 Jan 2018 19:58:09 +0100 Subject: [PATCH 47/48] Set default startup_time to 0 --- jupyter_client/restarter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_client/restarter.py b/jupyter_client/restarter.py index d7fbaa35e..103537386 100644 --- a/jupyter_client/restarter.py +++ b/jupyter_client/restarter.py @@ -36,7 +36,7 @@ class KernelRestarter(LoggingConfigurable): help="""Kernel heartbeat interval in seconds.""" ) - startup_time = Float(20.0, config=True, + startup_time = Float(0.0, config=True, help="""Waiting time for kernel_info reply during initial startup. 0 indicates that kernel_info reply check disabled.""" ) From 899ec691a3f1f1912bae24e023f1fde79d2996ab Mon Sep 17 00:00:00 2001 From: Sergey Kukhtichev Date: Wed, 31 Jan 2018 15:24:27 +0100 Subject: [PATCH 48/48] Updated logic for kernel monitor --- jupyter_client/restarter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter_client/restarter.py b/jupyter_client/restarter.py index 103537386..255b16b22 100644 --- a/jupyter_client/restarter.py +++ b/jupyter_client/restarter.py @@ -136,8 +136,8 @@ def poll(self): def is_kernel_response_timedout(self): """ - Sends kernel_info request and checks a response. If the response is not received within kernel info timeout - then returns True, and sets marks kernel as _restarting. If the response received or kernel startup timeout + The method sends kernel_info request and checks a response. If the response is not received within kernel info timeout + then returns True, and sets _restarting to True. If the response received or kernel startup timeout is not expired then returns False. :return: bool """