Merge pull request #29 from janschulz/new_activate

Updates so that env loading actually works

Author: Cadair

environment_kernels/core.py

@@ -5,7 +5,7 @@
 import os.path
 
 from jupyter_client.kernelspec import (KernelSpecManager, NoSuchKernel)
-from traitlets import List, Unicode, Bool
+from traitlets import List, Unicode, Bool, Int
 
 from .envs_conda import get_conda_env_data
 from .envs_virtualenv import get_virtualenv_env_data
@@ -33,8 +33,7 @@ class EnvironmentKernelSpecManager(KernelSpecManager):
 
     # Check for the CONDA_ENV_PATH variable and add it to the list if set.
     if os.environ.get('CONDA_ENV_PATH', False):
-        _default_conda_dirs.append(os.environ['CONDA_ENV_PATH'].split('envs')[
-            0])
+         _default_conda_dirs.append(os.environ['CONDA_ENV_PATH'].split('envs')[0])
 
     # If we are running inside the root conda env can get all the env dirs:
     if HAVE_CONDA:
@@ -95,13 +94,34 @@ class EnvironmentKernelSpecManager(KernelSpecManager):
         config=True,
         help="Probe for conda environments by calling conda itself. Only relevant if find_conda_envs is True.")
 
+    refresh_interval = Int(
+        3,
+        config=True,
+        help="Interval (in minutes) to refresh the list of environment kernels. Setting it to '0' disables the refresh.")
+
     find_virtualenv_envs = Bool(True,
                                 config=True,
                                 help="Probe for virtualenv environments.")
 
     def __init__(self, *args, **kwargs):
         super(EnvironmentKernelSpecManager, self).__init__(*args, **kwargs)
         self.log.info("Using EnvironmentKernelSpecManager...")
+        if self.refresh_interval > 0:
+            try:
+                from tornado.ioloop import PeriodicCallback, IOLoop
+                # Initial loading NOW
+                IOLoop.current().call_later(0, callback=self._update_env_data)
+                # Later updates
+                updater = PeriodicCallback(callback=self._update_env_data, callback_time=1000 * 60 * self.refresh_interval)
+                updater.start()
+                if not updater.is_running():
+                    raise Exception()
+                self._periodic_updater = updater
+                self.log.info("Started periodic updates of the kernel list (every %s minutes).", self.refresh_interval)
+            except:
+                self.log.exception("Error while trying to enable periodic updates of the kernel list.")
+        else:
+            self.log.info("Periodical updates the kernel list are DISABLED.")
 
     def validate_env(self, envname):
         """
@@ -121,6 +141,10 @@ def validate_env(self, envname):
         else:
             return True
 
+    def _update_env_data(self):
+        self.log.info("Starting periodic scan of virtual environments...")
+        self._get_env_data(reload=True)
+        self.log.debug("done...")
 
     def _get_env_data(self, reload=False):
         """Get the data about the available environments.
@@ -129,7 +153,7 @@ def _get_env_data(self, reload=False):
         """
 
         # This is called much too often and finding-process is really expensive :-(
-        if hasattr(self, "_env_data_cache"):
+        if not reload and hasattr(self, "_env_data_cache"):
             return getattr(self, "_env_data_cache")
 
         env_data = {}
@@ -138,12 +162,11 @@ def _get_env_data(self, reload=False):
 
         env_data = {name: env_data[name] for name in env_data if self.validate_env(name)}
         self.log.info("Found the following kernels in environments: %s",
-                          ", ".join(list(env_data)))
+                      ", ".join(list(env_data)))
 
         self._env_data_cache = env_data
         return env_data
 
-
     def find_kernel_specs_for_envs(self):
         """Returns a dict mapping kernel names to resource directories."""
         data = self._get_env_data()

environment_kernels/env_kernelspec.py

@@ -0,0 +1,40 @@
+# coding: utf-8
+"""Common function to deal with virtual environments"""
+from __future__ import absolute_import
+
+from jupyter_client.kernelspec import KernelSpec
+from traitlets import default
+
+_nothing = object()
+
+class EnvironmentLoadingKernelSpec(KernelSpec):
+    """A KernelSpec which loads `env` by activating the virtual environment"""
+
+    _loader = None
+    _env = _nothing
+
+    @property
+    def env(self):
+        if self._env is _nothing:
+            if self._loader:
+                try:
+                    self._env = self._loader()
+                except:
+                    self._env = {}
+        return self._env
+
+    def __init__(self, loader, **kwargs):
+        self._loader = loader
+        super(EnvironmentLoadingKernelSpec, self).__init__(**kwargs)
+
+
+    def to_dict(self):
+        d = dict(argv=self.argv,
+                 # Do not trigger the loading
+                 #env=self.env,
+                 display_name=self.display_name,
+                 language=self.language,
+                )
+
+        return d
+

environment_kernels/envs_common.py

@@ -6,11 +6,9 @@
 import os
 import glob
 
-from jupyter_client.kernelspec import KernelSpec
+from .env_kernelspec import EnvironmentLoadingKernelSpec
 
 
-from .lazyobj import LazyProxyDict
-
 def find_env_paths_in_basedirs(base_dirs):
     """Returns all potential envs in a basedir"""
     # get potential env path in the base_dirs
@@ -22,6 +20,7 @@ def find_env_paths_in_basedirs(base_dirs):
 
     return env_path
 
+
 def convert_to_env_data(mgr, env_paths, validator_func, activate_func,
                         name_template, display_name_template, name_prefix):
     """Converts a list of paths to environments to env_data.
@@ -51,15 +50,13 @@ def convert_to_env_data(mgr, env_paths, validator_func, activate_func,
 
         # the default vars are needed to save the vars in the function context
         def loader(env_dir=venv_dir, activate_func=activate_func, mgr=mgr):
-            return activate_func(mgr, env_dir)
-
-        # this constructs a proxy which only loads the real values when it is used.
-        # The result is that the env vars are only loaded (= env is activated) when
-        # the kernel is actually used, so we get a nice speedup on startup :-)
-        kspec_dict["env"] = LazyProxyDict(loader)
+            mgr.log.debug("Loading env data for %s" % env_dir)
+            res = activate_func(mgr, env_dir)
+            #mgr.log.info("PATH: %s" % res['PATH'])
+            return res
 
-        # This should probably use self.kernel_spec_class instead of the direct class
-        env_data.update({kernel_name: (resource_dir, KernelSpec(**kspec_dict))})
+        kspec = EnvironmentLoadingKernelSpec(loader, **kspec_dict)
+        env_data.update({kernel_name: (resource_dir, kspec)})
     return env_data
 
 
@@ -87,11 +84,11 @@ def validate_IPykernel(venv_dir):
     # check if this is really an ipython **kernel**
     import subprocess
     try:
-        subprocess.check_call([python_exe_name, '-c', '"IPython.kernel"'])
+        subprocess.check_call([python_exe_name, '-c', '"import ipykernel"'])
     except:
         # not installed? -> not useable in any case...
         return [], None, None
-    argv = [python_exe_name, "-m", "IPython.kernel", "-f", "{connection_file}"]
+    argv = [python_exe_name, "-m", "ipykernel", "-f", "{connection_file}"]
     resources_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "logos", "python")
     return argv, "python", resources_dir
 
@@ -134,5 +131,3 @@ def find_exe(env_dir, name):
             if not os.path.exists(exe_name):
                 return None
     return exe_name
-
-

environment_kernels/envs_conda.py

@@ -5,8 +5,7 @@
 from .activate_helper import source_env_vars_from_command
 from .envs_common import (find_env_paths_in_basedirs, convert_to_env_data,
                           validate_IPykernel, validate_IRkernel)
-from .utils import FileNotFoundError
-
+from .utils import FileNotFoundError, ON_WINDOWS
 
 def get_conda_env_data(mgr):
     """Finds kernel specs from conda environments
@@ -43,9 +42,15 @@ def get_conda_env_data(mgr):
 
 
 def _get_env_vars_for_conda_env(mgr, env_path):
-    args = ["activate", env_path]
+    if ON_WINDOWS:
+        args = ['activate', env_path]
+    else:
+        args = ['source', 'activate', env_path]
+
     try:
-        return source_env_vars_from_command(args)
+        envs = source_env_vars_from_command(args)
+        #mgr.log.debug("PATH: %s", envs['PATH'])
+        return envs
     except:
         # as a fallback, don't activate...
         mgr.log.exception(

environment_kernels/envs_virtualenv.py

@@ -37,9 +37,11 @@ def _get_env_vars_for_virtualenv_env(mgr, env_path):
     if ON_WINDOWS:
         args = [os.path.join(env_path, "Shell", "activate")]
     else:
-        args = [os.path.join(env_path, "bin", "activate")]
+        args = ['source', os.path.join(env_path, "bin", "activate")]
     try:
-        return source_env_vars_from_command(args)
+        envs = source_env_vars_from_command(args)
+        #mgr.log.debug("Environment variables: %s", envs)
+        return envs
     except:
         # as a fallback, don't activate...
         mgr.log.exception(