Merge pull request #708 from nf-core/dev

Version 1.10.1 patch release

Author: ewels

.github/workflows/create-lint-wf.yml

@@ -27,11 +27,17 @@ jobs:
 
       - name: Run nf-core/tools
         run: |
-          nf-core create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface"
-          nf-core lint nf-core-testpipeline
-          nf-core list
-          nf-core licences nf-core-testpipeline
-          nf-core sync nf-core-testpipeline/
-          nf-core schema build nf-core-testpipeline/ --no-prompts
-          nf-core bump-version nf-core-testpipeline/ 1.1
-          nf-core modules install nf-core-testpipeline/ fastqc
+          nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface"
+          nf-core --log-file log.txt lint nf-core-testpipeline
+          nf-core --log-file log.txt list
+          nf-core --log-file log.txt licences nf-core-testpipeline
+          nf-core --log-file log.txt sync nf-core-testpipeline/
+          nf-core --log-file log.txt schema build nf-core-testpipeline/ --no-prompts
+          nf-core --log-file log.txt bump-version nf-core-testpipeline/ 1.1
+          nf-core --log-file log.txt modules install nf-core-testpipeline/ fastqc
+
+      - name: Upload log file artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: nf-core-log-file
+          path: log.txt

.github/workflows/sync.yml

@@ -29,13 +29,17 @@ jobs:
           wget -qO- get.nextflow.io | bash
           sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow
 
-      - name: Configure git
-        run: |
-          git config user.email "[email protected]"
-          git config user.name "nf-core-bot"
-
       - name: Run synchronisation
         if: github.repository == 'nf-core/tools'
         env:
           AUTH_TOKEN: ${{ secrets.nf_core_bot_auth_token }}
-        run: nf-core sync --all --username nf-core-bot --auth-token $AUTH_TOKEN
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "nf-core-bot"
+          nf-core --log-file sync_log.txt sync --all --username nf-core-bot --auth-token $AUTH_TOKEN
+
+      - name: Upload sync log file artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: sync-log-file
+          path: sync_log.txt

CHANGELOG.md

@@ -1,5 +1,14 @@
 # nf-core/tools: Changelog
 
+## [v1.10.1](https://github.com/nf-core/tools/releases/tag/1.10.1) - [2020-07-30]
+
+Patch release to fix the automatic template synchronisation, which failed in the v1.10 release.
+
+* Improved logging: `nf-core --log-file log.txt` now saves a verbose log to disk.
+* GitHub actions sync now uploads verbose log as an artifact.
+* Sync - fixed several minor bugs, improved logging.
+* Python Rich library updated to `>=4.2.1`
+
 ## [v1.10 - Copper Camel](https://github.com/nf-core/tools/releases/tag/1.10) - [2020-07-30]
 
 ### Pipeline schema

nf_core/__main__.py

@@ -24,7 +24,9 @@
 import nf_core.sync
 import nf_core.utils
 
-log = logging.getLogger(__name__)
+# Set up logging as the root logger
+# Submodules should all traverse back to this
+log = logging.getLogger()
 
 
 def run_nf_core():
@@ -101,16 +103,30 @@ def decorator(f):
 
 @click.group(cls=CustomHelpOrder)
 @click.version_option(nf_core.__version__)
-@click.option("-v", "--verbose", is_flag=True, default=False, help="Verbose output (print debug statements).")
-def nf_core_cli(verbose):
-    stderr = rich.console.Console(file=sys.stderr)
-    logging.basicConfig(
-        level=logging.DEBUG if verbose else logging.INFO,
-        format="%(message)s",
-        datefmt=" ",
-        handlers=[rich.logging.RichHandler(console=stderr, markup=True)],
+@click.option("-v", "--verbose", is_flag=True, default=False, help="Print verbose output to the console.")
+@click.option("-l", "--log-file", help="Save a verbose log to a file.", metavar="<filename>")
+def nf_core_cli(verbose, log_file):
+
+    # Set the base logger to output DEBUG
+    log.setLevel(logging.DEBUG)
+
+    # Set up logs to the console
+    log.addHandler(
+        rich.logging.RichHandler(
+            level=logging.DEBUG if verbose else logging.INFO,
+            console=rich.console.Console(file=sys.stderr),
+            show_time=False,
+            markup=True,
+        )
     )
 
+    # Set up logs to a file if we asked for one
+    if log_file:
+        log_fh = logging.FileHandler(log_file, encoding="utf-8")
+        log_fh.setLevel(logging.DEBUG)
+        log_fh.setFormatter(logging.Formatter("[%(asctime)s] %(name)-20s [%(levelname)-7s]  %(message)s"))
+        log.addHandler(log_fh)
+
 
 # nf-core list
 @nf_core_cli.command(help_priority=1)
@@ -274,9 +290,10 @@ def create(name, description, author, new_version, no_git, force, outdir):
     and not os.environ.get("GITHUB_REPOSITORY", "") == "nf-core/tools",
     help="Execute additional checks for release-ready workflows.",
 )
+@click.option("-p", "--show-passed", is_flag=True, help="Show passing tests on the command line.")
 @click.option("--markdown", type=str, metavar="<filename>", help="File to write linting results to (Markdown)")
 @click.option("--json", type=str, metavar="<filename>", help="File to write linting results to (JSON)")
-def lint(pipeline_dir, release, markdown, json):
+def lint(pipeline_dir, release, show_passed, markdown, json):
     """
     Check pipeline code against nf-core guidelines.
 
@@ -286,7 +303,7 @@ def lint(pipeline_dir, release, markdown, json):
     """
 
     # Run the lint tests!
-    lint_obj = nf_core.lint.run_linting(pipeline_dir, release, markdown, json)
+    lint_obj = nf_core.lint.run_linting(pipeline_dir, release, show_passed, markdown, json)
     if len(lint_obj.failed) > 0:
         sys.exit(1)
 

nf_core/lint.py

@@ -39,7 +39,7 @@
 logging.getLogger("urllib3").setLevel(logging.WARNING)
 
 
-def run_linting(pipeline_dir, release_mode=False, md_fn=None, json_fn=None):
+def run_linting(pipeline_dir, release_mode=False, show_passed=False, md_fn=None, json_fn=None):
     """Runs all nf-core linting checks on a given Nextflow pipeline project
     in either `release` mode or `normal` mode (default). Returns an object
     of type :class:`PipelineLint` after finished.
@@ -65,7 +65,7 @@ def run_linting(pipeline_dir, release_mode=False, md_fn=None, json_fn=None):
         return lint_obj
 
     # Print the results
-    lint_obj.print_results()
+    lint_obj.print_results(show_passed)
 
     # Save results to Markdown file
     if md_fn is not None:
@@ -238,6 +238,7 @@ def lint_pipeline(self, release_mode=False):
             )
             for fun_name in check_functions:
                 progress.update(lint_progress, advance=1, func_name=fun_name)
+                log.debug("Running lint test: {}".format(fun_name))
                 getattr(self, fun_name)()
                 if len(self.failed) > 0:
                     log.error("Found test failures in `{}`, halting lint run.".format(fun_name))
@@ -1225,8 +1226,7 @@ def check_schema_lint(self):
         """ Lint the pipeline schema """
 
         # Only show error messages from schema
-        if log.getEffectiveLevel() == logging.INFO:
-            logging.getLogger("nf_core.schema").setLevel(logging.ERROR)
+        logging.getLogger("nf_core.schema").setLevel(logging.ERROR)
 
         # Lint the schema
         self.schema_obj = nf_core.schema.PipelineSchema()
@@ -1273,7 +1273,9 @@ def check_schema_params(self):
         if len(removed_params) == 0 and len(added_params) == 0:
             self.passed.append((15, "Schema matched params returned from nextflow config"))
 
-    def print_results(self):
+    def print_results(self, show_passed=False):
+
+        log.debug("Printing final results")
         console = Console()
 
         # Helper function to format test links nicely
@@ -1294,7 +1296,7 @@ def _s(some_list):
             return ""
 
         # Table of passed tests
-        if len(self.passed) > 0 and log.getEffectiveLevel() == logging.DEBUG:
+        if len(self.passed) > 0 and show_passed:
             table = Table(style="green", box=rich.box.ROUNDED)
             table.add_column(
                 "[[\u2714]] {} Test{} Passed".format(len(self.passed), _s(self.passed)), no_wrap=True,

nf_core/sync.py

@@ -100,7 +100,7 @@ def sync(self):
                 self.make_pull_request()
             except PullRequestException as e:
                 self.reset_target_dir()
-                raise PullRequestException(pr_exception)
+                raise PullRequestException(e)
 
         self.reset_target_dir()
 
@@ -140,7 +140,7 @@ def get_wf_config(self):
         # Try to check out target branch (eg. `origin/dev`)
         try:
             if self.from_branch and self.repo.active_branch.name != self.from_branch:
-                log.info("Checking out workflow branch '{}'".format(self.from_branch))
+                log.debug("Checking out workflow branch '{}'".format(self.from_branch))
                 self.repo.git.checkout(self.from_branch)
         except git.exc.GitCommandError:
             raise SyncException("Branch `{}` not found!".format(self.from_branch))
@@ -173,7 +173,7 @@ def get_wf_config(self):
             )
 
         # Fetch workflow variables
-        log.info("Fetching workflow config variables")
+        log.debug("Fetching workflow config variables")
         self.wf_config = nf_core.utils.fetch_wf_config(self.pipeline_dir)
 
         # Check that we have the required variables
@@ -201,7 +201,7 @@ def delete_template_branch_files(self):
         Delete all files in the TEMPLATE branch
         """
         # Delete everything
-        log.info("Deleting all files in TEMPLATE branch")
+        log.debug("Deleting all files in TEMPLATE branch")
         for the_file in os.listdir(self.pipeline_dir):
             if the_file == ".git":
                 continue
@@ -219,11 +219,10 @@ def make_template_pipeline(self):
         """
         Delete all files and make a fresh template using the workflow variables
         """
-        log.info("Making a new template pipeline using pipeline variables")
+        log.debug("Making a new template pipeline using pipeline variables")
 
         # Only show error messages from pipeline creation
-        if log.getEffectiveLevel() == logging.INFO:
-            logging.getLogger("nf_core.create").setLevel(logging.ERROR)
+        logging.getLogger("nf_core.create").setLevel(logging.ERROR)
 
         nf_core.create.PipelineCreate(
             name=self.wf_config["manifest.name"].strip('"').strip("'"),
@@ -247,7 +246,7 @@ def commit_template_changes(self):
             self.repo.git.add(A=True)
             self.repo.index.commit("Template update for nf-core/tools version {}".format(nf_core.__version__))
             self.made_changes = True
-            log.info("Committed changes to TEMPLATE branch")
+            log.debug("Committed changes to TEMPLATE branch")
         except Exception as e:
             raise SyncException("Could not commit changes to TEMPLATE:\n{}".format(e))
         return True
@@ -257,7 +256,7 @@ def push_template_branch(self):
         and try to make a PR. If we don't have the auth token, try to figure out a URL
         for the PR and print this to the console.
         """
-        log.info("Pushing TEMPLATE branch to remote")
+        log.debug("Pushing TEMPLATE branch to remote: '{}'".format(os.path.basename(self.pipeline_dir)))
         try:
             self.repo.git.push()
         except git.exc.GitCommandError as e:
@@ -287,23 +286,18 @@ def make_pull_request(self):
             )
             raise PullRequestException("No GitHub authentication token set - cannot make PR")
 
-        log.info("Submitting a pull request via the GitHub API")
+        log.debug("Submitting a pull request via the GitHub API")
 
-        pr_body_text = """
-            A new release of the main template in nf-core/tools has just been released.
-            This automated pull-request attempts to apply the relevant updates to this pipeline.
-
-            Please make sure to merge this pull-request as soon as possible.
-            Once complete, make a new minor release of your pipeline.
-
-            For instructions on how to merge this PR, please see
-            [https://nf-co.re/developers/sync](https://nf-co.re/developers/sync#merging-automated-prs).
-
-            For more information about this release of [nf-core/tools](https://github.com/nf-core/tools),
-            please see the [nf-core/tools v{tag} release page](https://github.com/nf-core/tools/releases/tag/{tag}).
-            """.format(
-            tag=nf_core.__version__
-        )
+        pr_body_text = (
+            "A new release of the main template in nf-core/tools has just been released. "
+            "This automated pull-request attempts to apply the relevant updates to this pipeline.\n\n"
+            "Please make sure to merge this pull-request as soon as possible. "
+            "Once complete, make a new minor release of your pipeline. "
+            "For instructions on how to merge this PR, please see "
+            "[https://nf-co.re/developers/sync](https://nf-co.re/developers/sync#merging-automated-prs).\n\n"
+            "For more information about this release of [nf-core/tools](https://github.com/nf-core/tools), "
+            "please see the [nf-core/tools v{tag} release page](https://github.com/nf-core/tools/releases/tag/{tag})."
+        ).format(tag=nf_core.__version__)
 
         pr_content = {
             "title": "Important! Template update for nf-core/tools v{}".format(nf_core.__version__),
@@ -360,6 +354,7 @@ def sync_all_pipelines(gh_username=None, gh_auth_token=None):
     # Let's do some updating!
     for wf in wfs.remote_workflows:
 
+        log.info("-" * 30)
         log.info("Syncing {}".format(wf.full_name))
 
         # Make a local working directory
@@ -373,8 +368,7 @@ def sync_all_pipelines(gh_username=None, gh_auth_token=None):
         assert repo
 
         # Only show error messages from pipeline creation
-        if log.getEffectiveLevel() == logging.INFO:
-            logging.getLogger("nf_core.create").setLevel(logging.ERROR)
+        logging.getLogger("nf_core.create").setLevel(logging.ERROR)
 
         # Sync the repo
         log.debug("Running template sync")
@@ -395,7 +389,7 @@ def sync_all_pipelines(gh_username=None, gh_auth_token=None):
             failed_syncs.append(wf.name)
         else:
             log.info(
-                "[green]Sync successful for {}:[/] [blue][link={1}]{1}[/link]".format(
+                "[green]Sync successful for {0}:[/] [blue][link={1}]{1}[/link]".format(
                     wf.full_name, sync_obj.gh_pr_returned_data.get("html_url")
                 )
             )

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 import sys
 
-version = "1.10"
+version = "1.10.1"
 
 with open("README.md") as f:
     readme = f.read()
@@ -40,7 +40,7 @@
         "pyyaml",
         "requests",
         "requests_cache",
-        "rich>=4.0.0",
+        "rich>=4.2.1",
         "tabulate",
     ],
     setup_requires=["twine>=1.11.0", "setuptools>=38.6."],