Merge pull request #2349 from nf-core/dev

Dev -> Master for v2.9

Author: mirpedrol

.github/CONTRIBUTING.md

@@ -142,4 +142,3 @@ To get started:
 Devcontainer specs:
 
 - [DevContainer config](.devcontainer/devcontainer.json)
-- [Dockerfile](.devcontainer/Dockerfile)

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -33,7 +33,7 @@ body:
     attributes:
       label: System information
       description: |
-        * Nextflow version _(eg. 22.10.1)_
+        * Nextflow version _(eg. 23.04.0)_
         * Hardware _(eg. HPC, Desktop, Cloud)_
         * Executor _(eg. slurm, local, awsbatch)_
         * OS _(eg. CentOS Linux, macOS, Linux Mint)_

.github/workflows/create-lint-wf.yml

@@ -23,7 +23,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "22.10.1"
+          - "23.04.0"
           - "latest-everything"
     steps:
       # Get the repo code

.github/workflows/create-test-lint-wf-template.yml

@@ -0,0 +1,137 @@
+name: Create a pipeline from a template and test it
+on:
+  push:
+    branches:
+      - dev
+    paths:
+      - nf_core/pipeline-template/**
+  pull_request:
+  release:
+    types: [published]
+
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  NXF_ANSI_LOG: false
+
+jobs:
+  RunTestWorkflow:
+    runs-on: ubuntu-latest
+    env:
+      NXF_ANSI_LOG: false
+    strategy:
+      matrix:
+        TEMPLATE:
+          - "template_skip_all.yml"
+          - "template_skip_github_badges.yml"
+          - "template_skip_igenomes.yml"
+          - "template_skip_ci.yml"
+          - "template_skip_nf_core_configs.yml"
+
+    steps:
+      - uses: actions/checkout@v3
+        name: Check out source-code repository
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+
+      - name: Install python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .
+
+      - name: Install Nextflow
+        uses: nf-core/setup-nextflow@v1
+        with:
+          version: latest-everything
+
+      # Install the Prettier linting tools
+      - uses: actions/setup-node@v3
+
+      - name: Install Prettier
+        run: npm install -g prettier
+
+      # Install the editorconfig linting tools
+      - name: Install editorconfig-checker
+        run: npm install -g editorconfig-checker
+
+      # Create template files
+      - name: Create template skip all (except github)
+        run: |
+          printf "prefix: my-prefix\nskip: ['ci', 'github_badges', 'igenomes', 'nf_core_configs']" > template_skip_all.yml
+
+      - name: Create template skip github_badges
+        run: |
+          printf "prefix: my-prefix\nskip: github_badges" > template_skip_github_badges.yml
+
+      - name: Create template skip igenomes
+        run: |
+          printf "prefix: my-prefix\nskip: igenomes" > template_skip_igenomes.yml
+
+      - name: Create template skip ci
+        run: |
+          printf "prefix: my-prefix\nskip: ci" > template_skip_ci.yml
+
+      - name: Create template skip nf_core_configs
+        run: |
+          printf "prefix: my-prefix\nskip: nf_core_configs" > template_skip_nf_core_configs.yml
+
+      # Create a pipeline from the template
+      - name: create a pipeline from the template ${{ matrix.TEMPLATE }}
+        run: |
+          nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --template-yaml ${{ matrix.TEMPLATE }}
+
+      - name: run the pipeline
+        run: |
+          nextflow run my-prefix-testpipeline -profile test,docker --outdir ./results
+
+      # Remove results folder before linting
+      - name: remove results folder
+        run: |
+          rm -rf ./results
+
+      # Try syncing it before we change anything
+      - name: nf-core sync
+        run: nf-core --log-file log.txt sync --dir my-prefix-testpipeline/ --template-yaml ${{ matrix.TEMPLATE }}
+
+      # Run code style linting
+      - name: Run Prettier --check
+        run: prettier --check my-prefix-testpipeline
+
+      - name: Run ECLint check
+        run: editorconfig-checker -exclude README.md $(find my-prefix-testpipeline/.* -type f | grep -v '.git\|.py\|md\|json\|yml\|yaml\|html\|css\|work\|.nextflow\|build\|nf_core.egg-info\|log.txt\|Makefile')
+
+      # Remove TODO statements
+      - name: remove TODO
+        run: find my-prefix-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \;
+
+      # Replace zenodo.XXXXXX to pass readme linting
+      - name: replace zenodo.XXXXXX
+        run: find my-prefix-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;
+
+      # Run nf-core linting
+      - name: nf-core lint
+        run: nf-core --log-file log.txt --hide-progress lint --dir my-prefix-testpipeline --fail-warned
+
+      # Run bump-version
+      - name: nf-core bump-version
+        run: nf-core --log-file log.txt bump-version --dir my-prefix-testpipeline/ 1.1
+
+      # Run nf-core linting in release mode
+      - name: nf-core lint in release mode
+        run: nf-core --log-file log.txt --hide-progress lint --dir my-prefix-testpipeline --fail-warned --release
+
+      - name: Tar files
+        run: tar -cvf artifact_files.tar log.txt template_skip*.yml
+
+      - name: Upload log file artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: nf-core-log-file
+          path: artifact_files.tar

.github/workflows/create-test-wf.yml

@@ -23,7 +23,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "22.10.1"
+          - "23.04.0"
           - "latest-everything"
     steps:
       - uses: actions/checkout@v3

.github/workflows/pytest-frozen-ubuntu-20.04.yml

@@ -15,7 +15,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  pytest:
+  pytest-frozen:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3

.gitpod.yml

@@ -5,6 +5,7 @@ tasks:
       python -m pip install -e .
       python -m pip install -r requirements-dev.txt
       pre-commit install --install-hooks
+      nextflow self-update
 vscode:
   extensions: # based on nf-core.nf-core-extensionpack
     - codezombiech.gitignore # Language support for .gitignore files

CHANGELOG.md

@@ -1,5 +1,73 @@
 # nf-core/tools: Changelog
 
+# [v2.9 - Chromium Falcon](https://github.com/nf-core/tools/releases/tag/2.9) + [2023-06-29]
+
+### Template
+
+- `params.max_multiqc_email_size` is no longer required ([#2273](https://github.com/nf-core/tools/pull/2273))
+- Remove `cleanup = true` from `test_full.config` in pipeline template ([#2279](https://github.com/nf-core/tools/pull/2279))
+- Fix usage docs for specifying `params.yaml` ([#2279](https://github.com/nf-core/tools/pull/2279))
+- Added stub in modules template ([#2277])(https://github.com/nf-core/tools/pull/2277) [Contributed by @nvnieuwk]
+- Move registry definitions out of profile scope ([#2286])(https://github.com/nf-core/tools/pull/2286)
+- Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287)
+- Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291))
+- Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298))
+- Updates seqeralabs/action-tower-launch to v2.0.0 ([#2301](https://github.com/nf-core/tools/pull/2301))
+- Remove schema validation from `lib` folder and use Nextflow [nf-validation plugin](https://nextflow-io.github.io/nf-validation/) instead ([#1771](https://github.com/nf-core/tools/pull/1771/))
+- Fix parsing of container directive when it is not typical nf-core format ([#2306](https://github.com/nf-core/tools/pull/2306))
+- Add ability to specify custom registry for linting modules, defaults to quay.io ([#2313](https://github.com/nf-core/tools/pull/2313))
+- Add `singularity.registry = 'quay.io'` in pipeline template ([#2305](https://github.com/nf-core/tools/pull/2305))
+- Add `apptainer.registry = 'quay.io'` in pipeline template ([#2352](https://github.com/nf-core/tools/pull/2352))
+- Bump minimum required NF version in pipeline template from `22.10.1` -> `23.04.0` ([#2305](https://github.com/nf-core/tools/pull/2305))
+- Add ability to interpret `docker.registry` from `nextflow.config` file. If not found defaults to quay.io. ([#2318](https://github.com/nf-core/tools/pull/2318))
+- Add functions to dynamically include pipeline tool citations in MultiQC methods description section for better reporting. ([#2326](https://github.com/nf-core/tools/pull/2326))
+- Remove `--tracedir` parameter ([#2290](https://github.com/nf-core/tools/pull/2290))
+- Incorrect config parameter warnings when customising pipeline template ([#2333](https://github.com/nf-core/tools/pull/2333))
+- Use markdown syntax in the description for the meta map channels ([#2358](https://github.com/nf-core/tools/pull/2358))
+
+### Download
+
+- Introduce a `--tower` flag for `nf-core download` to obtain pipelines in an offline format suited for [seqeralabs® Nextflow Tower](https://cloud.tower.nf/) ([#2247](https://github.com/nf-core/tools/pull/2247)).
+- Refactored the CLI for `--singularity-cache` in `nf-core download` from a flag to an argument. The prior options were renamed to `amend` (container images are only saved in the `$NXF_SINGULARITY_CACHEDIR`) and `copy` (a copy of the image is saved with the download). `remote` was newly introduced and allows to provide a table of contents of a remote cache via an additional argument `--singularity-cache-index` ([#2247](https://github.com/nf-core/tools/pull/2247)).
+- Refactored the CLI parameters related to container images. Although downloading other images than those of the Singularity/Apptainer container system is not supported for the time being, a generic name for the parameters seemed preferable. So the new parameter `--singularity-cache-index` introduced in [#2247](https://github.com/nf-core/tools/pull/2247) has been renamed to `--container-cache-index` prior to release ([#2336](https://github.com/nf-core/tools/pull/2336)).
+- To address issue [#2311](https://github.com/nf-core/tools/issues/2311), a new parameter `--container-library` was created allowing to specify the container library (registry) from which container images in OCI format (Docker) should be pulled ([#2336](https://github.com/nf-core/tools/pull/2336)).
+- Container detection in configs was improved. This allows for DSL2-like container definitions inside the container parameter value provided to process scopes [#2346](https://github.com/nf-core/tools/pull/2346).
+- Add apptainer to the list of false positve container strings ([#2353](https://github.com/nf-core/tools/pull/2353)).
+
+#### Updated CLI parameters
+
+| Old parameter         | New parameter                                  |
+| --------------------- | ---------------------------------------------- |
+| new parameter         | `-d` / `--download-configuration`              |
+| new parameter         | `-t` / `--tower`                               |
+| `-c`/ `--container`   | `-s` / `--container-system <VALUE>`            |
+| new parameter         | `-l` / `--container-library <VALUE>`           |
+| `--singularity-cache` | `-u` / `--container-cache-utilisation <VALUE>` |
+| new parameter         | `-i` / `--container-cache-index <VALUE>`       |
+
+_In addition, `-r` / `--revision` has been changed to a parameter that can be provided multiple times so several revisions can be downloaded at once._
+
+### Linting
+
+- Warn if container access is denied ([#2270](https://github.com/nf-core/tools/pull/2270))
+- Error if module container specification has quay.io as prefix when it shouldn't have ([#2278])(https://github.com/nf-core/tools/pull/2278/files)
+- Detect if container is 'simple name' and try to contact quay.io server by default ([#2281](https://github.com/nf-core/tools/pull/2281))
+- Warn about null/None/empty default values in `nextflow_schema.json` ([#3328](https://github.com/nf-core/tools/pull/2328))
+- Fix linting when creating a pipeline skipping some parts of the template and add CI test ([#2330](https://github.com/nf-core/tools/pull/2330))
+
+### Modules
+
+- Don't update `modules_json` object if a module is not updated ([#2323](https://github.com/nf-core/tools/pull/2323))
+
+### Subworkflows
+
+### General
+
+- GitPod base image: Always self-update to the latest version of Nextflow. Add [pre-commit](https://pre-commit.com/) dependency.
+- GitPod configs: Update Nextflow as an init task, init pre-commit in pipeline config.
+- Refgenie: Create `nxf_home/nf-core/refgenie_genomes.config` path if it doesn't exist ([#2312](https://github.com/nf-core/tools/pull/2312))
+- Add CI tests to test running a pipeline whe it's created from a template skipping different areas
+
 # [v2.8 - Ruthenium Monkey](https://github.com/nf-core/tools/releases/tag/2.8) - [2023-04-27]
 
 ### Template