Merge pull request #719 from nf-core/dev

ewels · web-flow · commit f14c7a5692c9 · 2020-07-31T12:49:25.000+02:00
1.10.2 Patch release
diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml
@@ -24,9 +24,7 @@ jobs:
           message: |
             Hi @${{ github.event.pull_request.user.login }},
 
-            It looks like this pull-request is has been made against the ${{github.event.pull_request.head.repo.full_name}} `master` branch.
-            The `master` branch on nf-core repositories should always contain code from the latest release.
-            Beacuse of this, PRs to `master` are only allowed if they come from the ${{github.event.pull_request.head.repo.full_name}} `dev` branch.
+            It looks like this pull-request has been made against the ${{github.event.pull_request.head.repo.full_name}} `master` branch. The `master` branch on nf-core repositories should always contain code from the latest release. Beacuse of this, PRs to `master` are only allowed if they come from the ${{github.event.pull_request.head.repo.full_name}} `dev` branch.
 
             You do not need to close this PR, you can change the target branch to `dev` by clicking the _"Edit"_ button at the top of this page.
 
diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml
@@ -4,13 +4,34 @@ on:
     types: [published]
 
 jobs:
-  sync-all:
-    name: Sync all pipelines
+  get-pipelines:
     runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - id: set-matrix
+        run: |
+          curl -O https://nf-co.re/pipeline_names.json
+          echo "::set-output name=matrix::$(cat pipeline_names.json)"
 
+  sync:
+    needs: get-pipelines
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}}
+      fail-fast: false
     steps:
+
       - uses: actions/checkout@v2
-        name: Check out source-code repository
+        name: Check out nf-core/tools
+
+      - uses: actions/checkout@v2
+        name: Check out nf-core/${{ matrix.pipeline }}
+        with:
+          repository: nf-core/${{ matrix.pipeline }}
+          ref: dev
+          token: ${{ secrets.nf_core_bot_auth_token }}
+          path: nf-core/${{ matrix.pipeline }}
 
       - name: Set up Python 3.8
         uses: actions/setup-python@v1
@@ -32,14 +53,20 @@ jobs:
       - name: Run synchronisation
         if: github.repository == 'nf-core/tools'
         env:
-          AUTH_TOKEN: ${{ secrets.nf_core_bot_auth_token }}
+          GITHUB_AUTH_TOKEN: ${{ secrets.nf_core_bot_auth_token }}
         run: |
           git config --global user.email "core@nf-co.re"
           git config --global user.name "nf-core-bot"
-          nf-core --log-file sync_log.txt sync --all --username nf-core-bot --auth-token $AUTH_TOKEN
+          nf-core --log-file sync_log_${{ matrix.pipeline }}.txt sync nf-core/${{ matrix.pipeline }} \
+            --from-branch dev \
+            --pull-request \
+            --username nf-core-bot \
+            --repository nf-core/${{ matrix.pipeline }}
+
 
       - name: Upload sync log file artifact
+        if: ${{ always() }}
         uses: actions/upload-artifact@v2
         with:
-          name: sync-log-file
-          path: sync_log.txt
+          name: sync_log_${{ matrix.pipeline }}
+          path: sync_log_${{ matrix.pipeline }}.txt
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,13 +1,32 @@
 # nf-core/tools: Changelog
 
-## [v1.10.1](https://github.com/nf-core/tools/releases/tag/1.10.1) - [2020-07-30]
+## [v1.10.2 - Copper Camel _(brought back from the dead)_](https://github.com/nf-core/tools/releases/tag/1.10.2) - [2020-07-31]
+
+Second patch release to address some small errors discovered in the pipeline template.
+Apologies for the inconvenience.
+
+* Fix syntax error in `/push_dockerhub.yml` GitHub Action workflow
+* Change `params.readPaths` -> `params.input_paths` in `test_full.config`
+* Check results when posting the lint results as a GitHub comment
+  * This feature is unfortunately not possible when making PRs from forks outside of the nf-core organisation for now.
+* More major refactoring of the automated pipeline sync
+  * New GitHub Actions matrix parallelisation of sync jobs across pipelines [[#673](https://github.com/nf-core/tools/issues/673)]
+  * Removed the `--all` behaviour from `nf-core sync` as we no longer need it
+  * Sync now uses a new list of pipelines on the website which does not include archived pipelines [[#712](https://github.com/nf-core/tools/issues/712)]
+  * When making a PR it checks if a PR already exists - if so it updates it [[#710](https://github.com/nf-core/tools/issues/710)]
+  * More tests and code refactoring for more stable code. Hopefully fixes 404 error [[#711](https://github.com/nf-core/tools/issues/711)]
+
+## [v1.10.1 - Copper Camel _(patch)_](https://github.com/nf-core/tools/releases/tag/1.10.1) - [2020-07-30]
 
 Patch release to fix the automatic template synchronisation, which failed in the v1.10 release.
 
 * Improved logging: `nf-core --log-file log.txt` now saves a verbose log to disk.
-* GitHub actions sync now uploads verbose log as an artifact.
-* Sync - fixed several minor bugs, improved logging.
+* nf-core/tools GitHub Actions pipeline sync now uploads verbose log as an artifact.
+* Sync - fixed several minor bugs, made logging less verbose.
 * Python Rich library updated to `>=4.2.1`
+* Hopefully fix git config for pipeline sync so that commit comes from @nf-core-bot
+* Fix sync auto-PR text indentation so that it doesn't all show as code
+* Added explicit flag `--show-passed` for `nf-core lint` instead of taking logging verbosity
 
 ## [v1.10 - Copper Camel](https://github.com/nf-core/tools/releases/tag/1.10) - [2020-07-30]
 
diff --git a/docs/lint_errors.md b/docs/lint_errors.md
@@ -177,7 +177,7 @@ This test will fail if the following requirements are not met in these files:
 
 2. `linting.yml`: Specifies the commands to lint the pipeline repository using `nf-core lint` and `markdownlint`
     * Must be turned on for `push` and `pull_request`.
-    * Must have the command `nf-core lint ${GITHUB_WORKSPACE}`.
+    * Must have the command `nf-core -l lint_log.txt lint ${GITHUB_WORKSPACE}`.
     * Must have the command `markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.github/markdownlint.yml`.
 
 3. `branch.yml`: Ensures that pull requests to the protected `master` branch are coming from the correct branch when a PR is opened against the _nf-core_ repository.
diff --git a/nf_core/__main__.py b/nf_core/__main__.py
@@ -550,14 +550,12 @@ def bump_version(pipeline_dir, new_version, nextflow):
 
 
 @nf_core_cli.command("sync", help_priority=10)
-@click.argument("pipeline_dir", type=click.Path(exists=True), nargs=-1, metavar="<pipeline directory>")
+@click.argument("pipeline_dir", required=True, type=click.Path(exists=True), metavar="<pipeline directory>")
 @click.option("-b", "--from-branch", type=str, help="The git branch to use to fetch workflow vars.")
 @click.option("-p", "--pull-request", is_flag=True, default=False, help="Make a GitHub pull-request with the changes.")
-@click.option("-u", "--username", type=str, help="GitHub username for the PR.")
-@click.option("-r", "--repository", type=str, help="GitHub repository name for the PR.")
-@click.option("-a", "--auth-token", type=str, help="GitHub API personal access token.")
-@click.option("--all", is_flag=True, default=False, help="Sync template for all nf-core pipelines.")
-def sync(pipeline_dir, from_branch, pull_request, username, repository, auth_token, all):
+@click.option("-r", "--repository", type=str, help="GitHub PR: target repository.")
+@click.option("-u", "--username", type=str, help="GitHub PR: auth username.")
+def sync(pipeline_dir, from_branch, pull_request, repository, username):
     """
     Sync a pipeline TEMPLATE branch with the nf-core template.
 
@@ -571,24 +569,13 @@ def sync(pipeline_dir, from_branch, pull_request, username, repository, auth_tok
     new release of nf-core/tools (and the included template) is made.
     """
 
-    # Pull and sync all nf-core pipelines
-    if all:
-        nf_core.sync.sync_all_pipelines(username, auth_token)
-    else:
-        # Manually check for the required parameter
-        if not pipeline_dir or len(pipeline_dir) != 1:
-            log.error("Either use --all or specify one <pipeline directory>")
-            sys.exit(1)
-        else:
-            pipeline_dir = pipeline_dir[0]
-
-        # Sync the given pipeline dir
-        sync_obj = nf_core.sync.PipelineSync(pipeline_dir, from_branch, pull_request)
-        try:
-            sync_obj.sync()
-        except (nf_core.sync.SyncException, nf_core.sync.PullRequestException) as e:
-            log.error(e)
-            sys.exit(1)
+    # Sync the given pipeline dir
+    sync_obj = nf_core.sync.PipelineSync(pipeline_dir, from_branch, pull_request, repository, username)
+    try:
+        sync_obj.sync()
+    except (nf_core.sync.SyncException, nf_core.sync.PullRequestException) as e:
+        log.error(e)
+        sys.exit(1)
 
 
 if __name__ == "__main__":
diff --git a/nf_core/lint.py b/nf_core/lint.py
@@ -771,7 +771,7 @@ def check_actions_lint(self):
                 self.passed.append((5, "Continuous integration runs Markdown lint Tests: `{}`".format(fn)))
 
             # Check that the nf-core linting runs
-            nfcore_lint_cmd = "nf-core lint ${GITHUB_WORKSPACE}"
+            nfcore_lint_cmd = "nf-core -l lint_log.txt lint ${GITHUB_WORKSPACE}"
             try:
                 steps = lintwf["jobs"]["nf-core"]["steps"]
                 assert any([nfcore_lint_cmd in step["run"] for step in steps if "run" in step.keys()])
@@ -1440,39 +1440,55 @@ def github_comment(self):
         """
         If we are running in a GitHub PR, try to post results as a comment
         """
-        if os.environ.get("GITHUB_TOKEN", "") != "" and os.environ.get("GITHUB_COMMENTS_URL", "") != "":
-            try:
-                headers = {"Authorization": "token {}".format(os.environ["GITHUB_TOKEN"])}
-                # Get existing comments - GET
-                get_r = requests.get(url=os.environ["GITHUB_COMMENTS_URL"], headers=headers)
-                if get_r.status_code == 200:
-
-                    # Look for an existing comment to update
-                    update_url = False
-                    for comment in get_r.json():
-                        if comment["user"]["login"] == "github-actions[bot]" and comment["body"].startswith(
-                            "\n#### `nf-core lint` overall result"
-                        ):
-                            # Update existing comment - PATCH
-                            log.info("Updating GitHub comment")
-                            update_r = requests.patch(
-                                url=comment["url"],
-                                data=json.dumps({"body": self.get_results_md().replace("Posted", "**Updated**")}),
-                                headers=headers,
-                            )
-                            return
-
-                    # Create new comment - POST
-                    if len(self.warned) > 0 or len(self.failed) > 0:
-                        log.info("Posting GitHub comment")
-                        post_r = requests.post(
-                            url=os.environ["GITHUB_COMMENTS_URL"],
-                            data=json.dumps({"body": self.get_results_md()}),
+        if os.environ.get("GITHUB_TOKEN", "") == "":
+            log.debug("Environment variable GITHUB_TOKEN not found")
+            return
+        if os.environ.get("GITHUB_COMMENTS_URL", "") == "":
+            log.debug("Environment variable GITHUB_COMMENTS_URL not found")
+            return
+        try:
+            headers = {"Authorization": "token {}".format(os.environ["GITHUB_TOKEN"])}
+            # Get existing comments - GET
+            get_r = requests.get(url=os.environ["GITHUB_COMMENTS_URL"], headers=headers)
+            if get_r.status_code == 200:
+
+                # Look for an existing comment to update
+                update_url = False
+                for comment in get_r.json():
+                    if comment["user"]["login"] == "github-actions[bot]" and comment["body"].startswith(
+                        "\n#### `nf-core lint` overall result"
+                    ):
+                        # Update existing comment - PATCH
+                        log.info("Updating GitHub comment")
+                        update_r = requests.patch(
+                            url=comment["url"],
+                            data=json.dumps({"body": self.get_results_md().replace("Posted", "**Updated**")}),
                             headers=headers,
                         )
+                        return
+
+                # Create new comment - POST
+                if len(self.warned) > 0 or len(self.failed) > 0:
+                    r = requests.post(
+                        url=os.environ["GITHUB_COMMENTS_URL"],
+                        data=json.dumps({"body": self.get_results_md()}),
+                        headers=headers,
+                    )
+                    try:
+                        r_json = json.loads(r.content)
+                        response_pp = json.dumps(r_json, indent=4)
+                    except:
+                        r_json = r.content
+                        response_pp = r.content
+
+                    if r.status_code == 201:
+                        log.info("Posted GitHub comment: {}".format(r_json["html_url"]))
+                        log.debug(response_pp)
+                    else:
+                        log.warn("Could not post GitHub comment: '{}'\n{}".format(r.status_code, response_pp))
 
-            except Exception as e:
-                log.warning("Could not post GitHub comment: {}\n{}".format(os.environ["GITHUB_COMMENTS_URL"], e))
+        except Exception as e:
+            log.warning("Could not post GitHub comment: {}\n{}".format(os.environ["GITHUB_COMMENTS_URL"], e))
 
     def _wrap_quotes(self, files):
         if not isinstance(files, list):
diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md
@@ -46,7 +46,7 @@ These tests are run both with the latest available version of `Nextflow` and als
 
 ## Patch
 
-: warning: Only in the unlikely and regretful event of a release happening with a bug.
+:warning: Only in the unlikely and regretful event of a release happening with a bug.
 
 * On your own fork, make a new branch `patch` based on `upstream/master`.
 * Fix the bug, and bump version (X.Y.Z+1).
diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/awsfulltest.yml
@@ -21,7 +21,7 @@ jobs:
         run: conda install -c conda-forge awscli
       - name: Start AWS batch job
         # TODO nf-core: You can customise AWS full pipeline tests as required
-        # Add full size test data (but still relatively small datasets for few samples) 
+        # Add full size test data (but still relatively small datasets for few samples)
         # on the `test_full.config` test runs with only one set of parameters
         # Then specify `-profile test_full` instead of `-profile test` on the AWS batch command
         {% raw %}env:
diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml
@@ -17,6 +17,7 @@ jobs:
 
 {% raw %}
       # If the above check failed, post a comment on the PR explaining the failure
+      # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets
       - name: Post PR comment
         if: failure()
         uses: mshick/add-pr-comment@v1
diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml
@@ -57,5 +57,12 @@ jobs:
           GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }}
-        run: nf-core lint ${GITHUB_WORKSPACE}
+        run: nf-core -l lint_log.txt lint ${GITHUB_WORKSPACE}
+
+      - name: Upload linting log file artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: linting-log-file
+          path: lint_log.txt
 {% endraw %}
diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/push_dockerhub.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/push_dockerhub.yml
@@ -8,32 +8,33 @@ on:
   release:
     types: [published]
 
-push_dockerhub:
-  name: Push new Docker image to Docker Hub
-  runs-on: ubuntu-latest
-  # Only run for the nf-core repo, for releases and merged PRs
-  if: {% raw %}${{{% endraw %} github.repository == '{{ cookiecutter.name }}' {% raw %}}}{% endraw %}
-  env:
-    DOCKERHUB_USERNAME: {% raw %}${{ secrets.DOCKERHUB_USERNAME }}{% endraw %}
-    DOCKERHUB_PASS: {% raw %}${{ secrets.DOCKERHUB_PASS }}{% endraw %}
-  steps:
-    - name: Check out pipeline code
-      uses: actions/checkout@v2
+jobs:
+  push_dockerhub:
+    name: Push new Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    # Only run for the nf-core repo, for releases and merged PRs
+    if: {% raw %}${{{% endraw %} github.repository == '{{ cookiecutter.name }}' {% raw %}}}{% endraw %}
+    env:
+      DOCKERHUB_USERNAME: {% raw %}${{ secrets.DOCKERHUB_USERNAME }}{% endraw %}
+      DOCKERHUB_PASS: {% raw %}${{ secrets.DOCKERHUB_PASS }}{% endraw %}
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@v2
 
-    - name: Build new docker image
-      run: docker build --no-cache . -t {{ cookiecutter.name_docker }}:latest
+      - name: Build new docker image
+        run: docker build --no-cache . -t {{ cookiecutter.name_docker }}:latest
 
-    - name: Push Docker image to DockerHub (dev)
-      if: {% raw %}${{ github.event_name == 'push' }}{% endraw %}
-      run: |
-        echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
-        docker tag {{ cookiecutter.name_docker }}:latest {{ cookiecutter.name_docker }}:dev
-        docker push {{ cookiecutter.name_docker }}:dev
+      - name: Push Docker image to DockerHub (dev)
+        if: {% raw %}${{ github.event_name == 'push' }}{% endraw %}
+        run: |
+          echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
+          docker tag {{ cookiecutter.name_docker }}:latest {{ cookiecutter.name_docker }}:dev
+          docker push {{ cookiecutter.name_docker }}:dev
 
-    - name: Push Docker image to DockerHub (release)
-      if: {% raw %}${{ github.event_name == 'release' }}{% endraw %}
-      run: |
-        echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
-        docker push {{ cookiecutter.name_docker }}:latest
-        docker tag {{ cookiecutter.name_docker }}:latest {{ cookiecutter.name_docker }}:{% raw %}${{ github.event.release.tag_name }}{% endraw %}
-        docker push {{ cookiecutter.name_docker }}:{% raw %}${{ github.event.release.tag_name }}{% endraw %}
+      - name: Push Docker image to DockerHub (release)
+        if: {% raw %}${{ github.event_name == 'release' }}{% endraw %}
+        run: |
+          echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
+          docker push {{ cookiecutter.name_docker }}:latest
+          docker tag {{ cookiecutter.name_docker }}:latest {{ cookiecutter.name_docker }}:{% raw %}${{ github.event.release.tag_name }}{% endraw %}
+          docker push {{ cookiecutter.name_docker }}:{% raw %}${{ github.event.release.tag_name }}{% endraw %}
diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test_full.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test_full.config
@@ -15,7 +15,7 @@ params {
   // TODO nf-core: Specify the paths to your full test data ( on nf-core/test-datasets or directly in repositories, e.g. SRA)
   // TODO nf-core: Give any required params for the test so that command line flags are not needed
   single_end = false
-  readPaths = [
+  input_paths = [
     ['Testdata', ['https://github.com/nf-core/test-datasets/raw/exoseq/testdata/Testdata_R1.tiny.fastq.gz', 'https://github.com/nf-core/test-datasets/raw/exoseq/testdata/Testdata_R2.tiny.fastq.gz']],
     ['SRR389222', ['https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', 'https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub2.fastq.gz']]
   ]
diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md
@@ -80,7 +80,7 @@ You can also supply a run name to resume a specific run: `-resume [run-name]`. U
 
 ### `-c`
 
-Specify the path to a specific config file (this is a core NextFlow command). See the [nf-core website documentation](https://nf-co.re/usage/configuration) for more information.
+Specify the path to a specific config file (this is a core Nextflow command). See the [nf-core website documentation](https://nf-co.re/usage/configuration) for more information.
 
 #### Custom resource requests
 
diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf
diff --git a/nf_core/sync.py b/nf_core/sync.py
diff --git a/setup.py b/setup.py
diff --git a/tests/lint_examples/minimalworkingexample/.github/workflows/linting.yml b/tests/lint_examples/minimalworkingexample/.github/workflows/linting.yml
diff --git a/tests/test_sync.py b/tests/test_sync.py
