Guard against inconsistently cased filesystems for @conda on macOS with @Batch (#377)

Conda fails to correctly set up environments for linux-64 packages on macOS which
is needed to collect the necessary metadata for correctly setting up the conda
environment on AWS Batch. This patch simply ignores the error-checks that conda
throws while setting up the environments on macOS when the intended destination
is AWS Batch.

Author: savingoyal

metaflow/plugins/conda/conda.py

@@ -41,12 +41,22 @@ def __init__(self):
                                               'is required. Specify it with CONDA_CHANNELS '
                                               'environment variable.')
 
-    def create(self, step_name, env_id, deps, architecture=None, explicit=False):
+    def create(self,
+               step_name,
+               env_id,
+               deps,
+               architecture=None,
+               explicit=False,
+               disable_safety_checks=False):
         # Create the conda environment
         try:
             with CondaLock(self._env_lock_file(env_id)):
                 self._remove(env_id)
-                self._create(env_id, deps, explicit, architecture)
+                self._create(env_id,
+                             deps,
+                             explicit,
+                             architecture,
+                             disable_safety_checks)
                 return self._deps(env_id)
         except CondaException as e:
             raise CondaStepException(e, step_name)
@@ -91,13 +101,20 @@ def package_info(self, env_id):
     def _info(self):
         return json.loads(self._call_conda(['info']))
 
-    def _create(self, env_id, deps, explicit=False, architecture=None):
+    def _create(self,
+                env_id,
+                deps,
+                explicit=False,
+                architecture=None,
+                disable_safety_checks=False):
         cmd = ['create', '--yes', '--no-default-packages',
                '--name', env_id, '--quiet']
         if explicit:
             cmd.append('--no-deps')
         cmd.extend(deps)
-        self._call_conda(cmd, architecture)
+        self._call_conda(cmd,
+                         architecture=architecture,
+                         disable_safety_checks=disable_safety_checks)
 
     def _remove(self, env_id):
         self._call_conda(['env', 'remove', '--name',
@@ -143,19 +160,19 @@ def _env_path(self, env_id):
     def _env_lock_file(self, env_id):
         return os.path.join(self._info()['envs_dirs'][0], 'mf_env-creation.lock')
 
-    def _call_conda(self, args, architecture=None):
+    def _call_conda(self, args, architecture=None, disable_safety_checks=False):
         try:
+            env = {
+                'CONDA_JSON': 'True',
+                'CONDA_SUBDIR': (architecture if architecture else ''),
+                'CONDA_USE_ONLY_TAR_BZ2': 'True'
+            }
+            if disable_safety_checks:
+                env['CONDA_SAFETY_CHECKS'] = 'disabled'
             return subprocess.check_output(
                 [self._bin] + args, 
                 stderr = open(os.devnull, 'wb'),
-                env = dict(
-                    os.environ,
-                    **{
-                        'CONDA_JSON': 'True',
-                        'CONDA_SUBDIR': (architecture if architecture else ''),
-                        'CONDA_USE_ONLY_TAR_BZ2': 'True'
-                    })
-                ).strip()
+                env = dict(os.environ, **env)).strip()
         except subprocess.CalledProcessError as e:
             try:
                 output = json.loads(e.output)
@@ -165,7 +182,7 @@ def _call_conda(self, args, architecture=None):
                 raise CondaException(err)
             except (TypeError, ValueError) as ve:
                 pass
-            raise RuntimeError(
+            raise CondaException(
                 'command \'{cmd}\' returned error ({code}): {output}'
                     .format(cmd=e.cmd, code=e.returncode, output=e.output))
 

metaflow/plugins/conda/conda_step_decorator.py

@@ -108,12 +108,17 @@ def _resolve_step_environment(self, ds_root, force=False):
         cached_deps = read_conda_manifest(ds_root, self.flow.name)
         if CondaStepDecorator.conda is None:
             CondaStepDecorator.conda = Conda()
-            CondaStepDecorator.environments = CondaStepDecorator.conda.environments(self.flow.name)
+            CondaStepDecorator.environments =\
+                CondaStepDecorator.conda.environments(self.flow.name)
         if force or env_id not in cached_deps or 'cache_urls' not in cached_deps[env_id]:
             if force or env_id not in cached_deps:
                 deps = self._step_deps()
                 (exact_deps, urls, order) = \
-                    self.conda.create(self.step, env_id, deps, architecture=self.architecture)
+                    self.conda.create(self.step,
+                                      env_id,
+                                      deps,
+                                      architecture=self.architecture,
+                                      disable_safety_checks=self.disable_safety_checks)
                 payload = {
                     'explicit': exact_deps,
                     'deps': [d.decode('ascii') for d in deps],
@@ -125,7 +130,8 @@ def _resolve_step_environment(self, ds_root, force=False):
             if self.datastore.TYPE == 's3' and 'cache_urls' not in payload:
                 payload['cache_urls'] = self._cache_env()
             write_to_conda_manifest(ds_root, self.flow.name, env_id, payload)
-            CondaStepDecorator.environments = CondaStepDecorator.conda.environments(self.flow.name)
+            CondaStepDecorator.environments =\
+                CondaStepDecorator.conda.environments(self.flow.name)
         return env_id
 
     def _cache_env(self):
@@ -147,13 +153,13 @@ def _download(entry):
                                 package_info['fn'])
             tarball_path = package_info['package_tarball_full_path']
             if tarball_path.endswith('.conda'):
-                #Conda doesn't set the metadata correctly for certain fields
+                # Conda doesn't set the metadata correctly for certain fields
                 # when the underlying OS is spoofed.
                 tarball_path = tarball_path[:-6]
             if not tarball_path.endswith('.tar.bz2'):
                 tarball_path = '%s.tar.bz2' % tarball_path
             if not os.path.isfile(tarball_path):
-                #The tarball maybe missing when user invokes `conda clean`!
+                # The tarball maybe missing when user invokes `conda clean`!
                 to_download.append((package_info['url'], tarball_path))
             files.append((path, tarball_path))
         if to_download:
@@ -170,10 +176,21 @@ def _prepare_step_environment(self, step_name, ds_root):
                               env_id,
                               cached_deps[env_id]['urls'],
                               architecture=self.architecture,
-                              explicit=True)
-            CondaStepDecorator.environments = CondaStepDecorator.conda.environments(self.flow.name)
+                              explicit=True,
+                              disable_safety_checks=self.disable_safety_checks)
+            CondaStepDecorator.environments =\
+                CondaStepDecorator.conda.environments(self.flow.name)
         return env_id
 
+    def _disable_safety_checks(self, decos):
+        # Disable conda safety checks when creating linux-64 environments on
+        # a macOS. This is needed because of gotchas around inconsistently 
+        # case-(in)sensitive filesystems for macOS and linux.
+        for deco in decos:
+            if deco.name == 'batch' and platform.system() == 'Darwin':
+                return True
+        return False
+
     def _architecture(self, decos):
         for deco in decos:
             if deco.name == 'batch':
@@ -205,6 +222,7 @@ def _logger(line, **kwargs):
         self.local_root = LocalDataStore.get_datastore_root_from_config(_logger)
         environment.set_local_root(self.local_root)
         self.architecture = self._architecture(decos)
+        self.disable_safety_checks = self._disable_safety_checks(decos)
         self.step = step
         self.flow = flow
         self.datastore = datastore