Merge pull request #806 from nf-core/dev

Dev > Master for release

Author: ewels

.github/markdownlint.yml

@@ -3,6 +3,13 @@ default: true
 line-length: false
 no-duplicate-header:
     siblings_only: true
+no-inline-html:
+    allowed_elements:
+        - img
+        - p
+        - kbd
+        - details
+        - summary
 # tools only - the {{ jinja variables }} break URLs and cause this to error
 no-bare-urls: false
 # tools only - suppresses error messages for usage of $ in main README

.gitignore

@@ -112,3 +112,5 @@ ENV/
 
 # Jetbrains IDEs
 .idea
+pip-wheel-metadata
+.vscode

CHANGELOG.md

@@ -1,5 +1,23 @@
 # nf-core/tools: Changelog
 
+## [v1.12.1 - Silver Dolphin](https://github.com/nf-core/tools/releases/tag/1.12.1) - [2020-12-03]
+
+### Template
+
+* Finished switch from `$baseDir` to `$projectDir` in `iGenomes.conf` and `main.nf`
+  * Main fix is for `smail_fields` which was a bug introduced in the previous release. Sorry about that!
+* Ported a number of small content tweaks from nf-core/eager to the template [[#786](https://github.com/nf-core/tools/issues/786)]
+  * Better contributing documentation, more placeholders in documentation files, more relaxed markdownlint exceptions for certain HTML tags, more content for the PR and issue templates.
+
+### Tools helper code
+
+* Pipeline schema: make parameters of type `range` to `number`. [[#738](https://github.com/nf-core/tools/issues/738)]
+* Respect `$NXF_HOME` when looking for pipelines with `nf-core list` [[#798](https://github.com/nf-core/tools/issues/798)]
+* Swapped PyInquirer with questionary for command line questions in `launch.py` [[#726](https://github.com/nf-core/tools/issues/726)]
+  * This should fix conda installation issues that some people had been hitting
+  * The change also allows other improvements to the UI
+* Fix linting crash when a file deleted but not yet staged in git [[#796](https://github.com/nf-core/tools/issues/796)]
+
 ## [v1.12 - Mercury Weasel](https://github.com/nf-core/tools/releases/tag/1.12) - [2020-11-19]
 
 ### Tools helper code

nf_core/launch.py

@@ -10,7 +10,8 @@
 import json
 import logging
 import os
-import PyInquirer
+import prompt_toolkit
+import questionary
 import re
 import subprocess
 import textwrap
@@ -20,15 +21,21 @@
 
 log = logging.getLogger(__name__)
 
-#
-# NOTE: When PyInquirer 1.0.3 is released we can capture keyboard interruptions
-# in a nicer way # with the raise_keyboard_interrupt=True argument in the PyInquirer.prompt() calls
-# It also allows list selections to have a default set.
-#
-# Until then we have workarounds:
-# * Default list item is moved to the top of the list
-# * We manually raise a KeyboardInterrupt if we get None back from a question
-#
+# Custom style for questionary
+nfcore_question_style = prompt_toolkit.styles.Style(
+    [
+        ("qmark", "fg:ansiblue bold"),  # token in front of the question
+        ("question", "bold"),  # question text
+        ("answer", "fg:ansigreen nobold"),  # submitted answer text behind the question
+        ("pointer", "fg:ansiyellow bold"),  # pointer used in select and checkbox prompts
+        ("highlighted", "fg:ansiblue bold"),  # pointed-at choice in select and checkbox prompts
+        ("selected", "fg:ansigreen noreverse"),  # style for a selected item of a checkbox
+        ("separator", "fg:ansiblack"),  # separator in lists
+        ("instruction", ""),  # user instructions for select, rawselect, checkbox
+        ("text", ""),  # plain text
+        ("disabled", "fg:gray italic"),  # disabled choices for select and checkbox prompts
+    ]
+)
 
 
 class Launch(object):
@@ -256,11 +263,9 @@ def prompt_web_gui(self):
             "name": "use_web_gui",
             "message": "Choose launch method",
             "choices": ["Web based", "Command line"],
+            "default": "Web based",
         }
-        answer = PyInquirer.prompt([question])
-        # TODO: use raise_keyboard_interrupt=True when PyInquirer 1.0.3 is released
-        if answer == {}:
-            raise KeyboardInterrupt
+        answer = questionary.unsafe_prompt([question], style=nfcore_question_style)
         return answer["use_web_gui"] == "Web based"
 
     def launch_web_gui(self):
@@ -347,14 +352,14 @@ def sanitise_web_response(self):
         The web builder returns everything as strings.
         Use the functions defined in the cli wizard to convert to the correct types.
         """
-        # Collect pyinquirer objects for each defined input_param
-        pyinquirer_objects = {}
+        # Collect questionary objects for each defined input_param
+        questionary_objects = {}
         for param_id, param_obj in self.schema_obj.schema.get("properties", {}).items():
-            pyinquirer_objects[param_id] = self.single_param_to_pyinquirer(param_id, param_obj, print_help=False)
+            questionary_objects[param_id] = self.single_param_to_questionary(param_id, param_obj, print_help=False)
 
         for d_key, definition in self.schema_obj.schema.get("definitions", {}).items():
             for param_id, param_obj in definition.get("properties", {}).items():
-                pyinquirer_objects[param_id] = self.single_param_to_pyinquirer(param_id, param_obj, print_help=False)
+                questionary_objects[param_id] = self.single_param_to_questionary(param_id, param_obj, print_help=False)
 
         # Go through input params and sanitise
         for params in [self.nxf_flags, self.schema_obj.input_params]:
@@ -364,7 +369,7 @@ def sanitise_web_response(self):
                     del params[param_id]
                     continue
                 # Run filter function on value
-                filter_func = pyinquirer_objects.get(param_id, {}).get("filter")
+                filter_func = questionary_objects.get(param_id, {}).get("filter")
                 if filter_func is not None:
                     params[param_id] = filter_func(params[param_id])
 
@@ -396,19 +401,13 @@ def prompt_param(self, param_id, param_obj, is_required, answers):
         """Prompt for a single parameter"""
 
         # Print the question
-        question = self.single_param_to_pyinquirer(param_id, param_obj, answers)
-        answer = PyInquirer.prompt([question])
-        # TODO: use raise_keyboard_interrupt=True when PyInquirer 1.0.3 is released
-        if answer == {}:
-            raise KeyboardInterrupt
+        question = self.single_param_to_questionary(param_id, param_obj, answers)
+        answer = questionary.unsafe_prompt([question], style=nfcore_question_style)
 
         # If required and got an empty reponse, ask again
         while type(answer[param_id]) is str and answer[param_id].strip() == "" and is_required:
             log.error("'–-{}' is required".format(param_id))
-            answer = PyInquirer.prompt([question])
-            # TODO: use raise_keyboard_interrupt=True when PyInquirer 1.0.3 is released
-            if answer == {}:
-                raise KeyboardInterrupt
+            answer = questionary.unsafe_prompt([question], style=nfcore_question_style)
 
         # Don't return empty answers
         if answer[param_id] == "":
@@ -426,37 +425,39 @@ def prompt_group(self, group_id, group_obj):
         Returns:
           Dict of param_id:val answers
         """
-        question = {
-            "type": "list",
-            "name": group_id,
-            "message": group_obj.get("title", group_id),
-            "choices": ["Continue >>", PyInquirer.Separator()],
-        }
-
-        for param_id, param in group_obj["properties"].items():
-            if not param.get("hidden", False) or self.show_hidden:
-                question["choices"].append(param_id)
-
-        # Skip if all questions hidden
-        if len(question["choices"]) == 2:
-            return {}
-
         while_break = False
         answers = {}
         while not while_break:
+            question = {
+                "type": "list",
+                "name": group_id,
+                "message": group_obj.get("title", group_id),
+                "choices": ["Continue >>", questionary.Separator()],
+            }
+
+            for param_id, param in group_obj["properties"].items():
+                if not param.get("hidden", False) or self.show_hidden:
+                    q_title = param_id
+                    if param_id in answers:
+                        q_title += "  [{}]".format(answers[param_id])
+                    elif "default" in param:
+                        q_title += "  [{}]".format(param["default"])
+                    question["choices"].append(questionary.Choice(title=q_title, value=param_id))
+
+            # Skip if all questions hidden
+            if len(question["choices"]) == 2:
+                return {}
+
             self.print_param_header(group_id, group_obj)
-            answer = PyInquirer.prompt([question])
-            # TODO: use raise_keyboard_interrupt=True when PyInquirer 1.0.3 is released
-            if answer == {}:
-                raise KeyboardInterrupt
+            answer = questionary.unsafe_prompt([question], style=nfcore_question_style)
             if answer[group_id] == "Continue >>":
                 while_break = True
                 # Check if there are any required parameters that don't have answers
                 for p_required in group_obj.get("required", []):
                     req_default = self.schema_obj.input_params.get(p_required, "")
                     req_answer = answers.get(p_required, "")
                     if req_default == "" and req_answer == "":
-                        log.error("'{}' is required.".format(p_required))
+                        log.error("'--{}' is required.".format(p_required))
                         while_break = False
             else:
                 param_id = answer[group_id]
@@ -465,8 +466,8 @@ def prompt_group(self, group_id, group_obj):
 
         return answers
 
-    def single_param_to_pyinquirer(self, param_id, param_obj, answers=None, print_help=True):
-        """Convert a JSONSchema param to a PyInquirer question
+    def single_param_to_questionary(self, param_id, param_obj, answers=None, print_help=True):
+        """Convert a JSONSchema param to a Questionary question
 
         Args:
           param_id: Parameter ID (string)
@@ -475,7 +476,7 @@ def single_param_to_pyinquirer(self, param_id, param_obj, answers=None, print_he
           print_help: If description and help_text should be printed (bool)
 
         Returns:
-          Single PyInquirer dict, to be appended to questions list
+          Single Questionary dict, to be appended to questions list
         """
         if answers is None:
             answers = {}
@@ -530,7 +531,11 @@ def validate_number(val):
                 try:
                     if val.strip() == "":
                         return True
-                    float(val)
+                    fval = float(val)
+                    if "minimum" in param_obj and fval < float(param_obj["minimum"]):
+                        return "Must be greater than or equal to {}".format(param_obj["minimum"])
+                    if "maximum" in param_obj and fval > float(param_obj["maximum"]):
+                        return "Must be less than or equal to {}".format(param_obj["maximum"])
                 except ValueError:
                     return "Must be a number"
                 else:
@@ -568,46 +573,11 @@ def filter_integer(val):
 
             question["filter"] = filter_integer
 
-        if param_obj.get("type") == "range":
-            # Validate range type
-            def validate_range(val):
-                try:
-                    if val.strip() == "":
-                        return True
-                    fval = float(val)
-                    if "minimum" in param_obj and fval < float(param_obj["minimum"]):
-                        return "Must be greater than or equal to {}".format(param_obj["minimum"])
-                    if "maximum" in param_obj and fval > float(param_obj["maximum"]):
-                        return "Must be less than or equal to {}".format(param_obj["maximum"])
-                    return True
-                except ValueError:
-                    return "Must be a number"
-
-            question["validate"] = validate_range
-
-            # Filter returned value
-            def filter_range(val):
-                if val.strip() == "":
-                    return ""
-                return float(val)
-
-            question["filter"] = filter_range
-
         if "enum" in param_obj:
             # Use a selection list instead of free text input
             question["type"] = "list"
             question["choices"] = param_obj["enum"]
 
-            # Validate enum from schema
-            def validate_enum(val):
-                if val == "":
-                    return True
-                if val in param_obj["enum"]:
-                    return True
-                return "Must be one of: {}".format(", ".join(param_obj["enum"]))
-
-            question["validate"] = validate_enum
-
         # Validate pattern from schema
         if "pattern" in param_obj:
 
@@ -620,21 +590,6 @@ def validate_pattern(val):
 
             question["validate"] = validate_pattern
 
-        # WORKAROUND - PyInquirer <1.0.3 cannot have a default position in a list
-        # For now, move the default option to the top.
-        # TODO: Delete this code when PyInquirer >=1.0.3 is released.
-        if question["type"] == "list" and "default" in question:
-            try:
-                question["choices"].remove(question["default"])
-                question["choices"].insert(0, question["default"])
-            except ValueError:
-                log.warning(
-                    "Default value `{}` not found in list of choices: {}".format(
-                        question["default"], ", ".join(question["choices"])
-                    )
-                )
-        ### End of workaround code
-
         return question
 
     def print_param_header(self, param_id, param_obj):

nf_core/lint.py

@@ -239,7 +239,7 @@ def lint_pipeline(self, release_mode=False):
                 log.debug("Running lint test: {}".format(fun_name))
                 getattr(self, fun_name)()
                 if len(self.failed) > 0:
-                    log.error("Found test failures in `{}`, halting lint run.".format(fun_name))
+                    log.critical("Found test failures in `{}`, halting lint run.".format(fun_name))
                     break
 
     def check_files_exist(self):
@@ -1241,17 +1241,25 @@ def check_cookiecutter_strings(self):
         num_files = 0
         for fn in list_of_files:
             num_files += 1
-            with io.open(fn, "r", encoding="latin1") as fh:
-                lnum = 0
-                for l in fh:
-                    lnum += 1
-                    cc_matches = re.findall(r"{{\s*cookiecutter[^}]*}}", l)
-                    if len(cc_matches) > 0:
-                        for cc_match in cc_matches:
-                            self.failed.append(
-                                (13, "Found a cookiecutter template string in `{}` L{}: {}".format(fn, lnum, cc_match))
-                            )
-                            num_matches += 1
+            try:
+                with io.open(fn, "r", encoding="latin1") as fh:
+                    lnum = 0
+                    for l in fh:
+                        lnum += 1
+                        cc_matches = re.findall(r"{{\s*cookiecutter[^}]*}}", l)
+                        if len(cc_matches) > 0:
+                            for cc_match in cc_matches:
+                                self.failed.append(
+                                    (
+                                        13,
+                                        "Found a cookiecutter template string in `{}` L{}: {}".format(
+                                            fn, lnum, cc_match
+                                        ),
+                                    )
+                                )
+                                num_matches += 1
+            except FileNotFoundError as e:
+                log.warn("`git ls-files` returned '{}' but could not open it!".format(fn))
         if num_matches == 0:
             self.passed.append((13, "Did not find any cookiecutter template strings ({} files)".format(num_files)))
 

nf_core/list.py

@@ -128,6 +128,8 @@ def get_local_nf_workflows(self):
         # Try to guess the local cache directory (much faster than calling nextflow)
         if len(os.environ.get("NXF_ASSETS", "")) > 0:
             nextflow_wfdir = os.environ.get("NXF_ASSETS")
+        elif len(os.environ.get("NXF_HOME", "")) > 0:
+            nextflow_wfdir = os.path.join(os.environ.get("NXF_HOME"), "assets")
         else:
             nextflow_wfdir = os.path.join(os.getenv("HOME"), ".nextflow", "assets")
         if os.path.isdir(nextflow_wfdir):
@@ -348,6 +350,8 @@ def get_local_nf_workflow_details(self):
             # Try to guess the local cache directory
             if len(os.environ.get("NXF_ASSETS", "")) > 0:
                 nf_wfdir = os.path.join(os.environ.get("NXF_ASSETS"), self.full_name)
+            elif len(os.environ.get("NXF_HOME", "")) > 0:
+                nf_wfdir = os.path.join(os.environ.get("NXF_HOME"), "assets")
             else:
                 nf_wfdir = os.path.join(os.getenv("HOME"), ".nextflow", "assets", self.full_name)
             if os.path.isdir(nf_wfdir):