diff --git a/.devcontainer/local-features/uv-tools/install.sh b/.devcontainer/local-features/uv-tools/install.sh index 8537c04973..256cf0ca48 100644 --- a/.devcontainer/local-features/uv-tools/install.sh +++ b/.devcontainer/local-features/uv-tools/install.sh @@ -3,5 +3,5 @@ # Install python cli tools using uv uv tool install pre-commit -uv tool install nf-core==3.5.2 +uv tool install nf-core==4.0.2 uv tool install "mkdocs-quiz>=1.5.2" diff --git a/docs/en/docs/hello_nf-core/00_orientation.md b/docs/en/docs/hello_nf-core/00_orientation.md index 836c42dc48..98113e39af 100644 --- a/docs/en/docs/hello_nf-core/00_orientation.md +++ b/docs/en/docs/hello_nf-core/00_orientation.md @@ -20,28 +20,20 @@ If you are working through this course by yourself, please acquaint yourself wit ### Version requirements -This training works with **Nextflow 25.10.2** or later and **requires the v1 syntax parser**, because the nf-core conventions used in this course are not yet compatible with the v2 parser. -From Nextflow 26.04 the v2 parser is the default, so you must explicitly select v1 as shown below. +This training works with Nextflow 25.10.2 or later **with the v2 syntax parser**, which is the default from Nextflow 26.04 onward. +In our training environment you don't need to do anything: it runs Nextflow 26.04.4 with the v2 parser. If you are using a local or custom environment, see the [version notes](../info/nxf_versions.md). -#### If you are using our training environment: - -You MUST run the following command before going any further: - -```bash -export NXF_SYNTAX_PARSER=v1 -``` - -On Nextflow 26.04 and later this is essential: it overrides the v2 parser that those versions enable by default (our training environment sets `NXF_SYNTAX_PARSER=v2` for the other courses). - -#### If you are using a local or custom environment: - -Please make sure you are using the correct settings as documented [here](../info/nxf_versions.md). - -The training additionally requires **nf-core tools 3.5.2**. +This training additionally requires **nf-core tools 4.0.2**. If you use a different version of nf-core tooling, you may have difficulty following along. You can check what version is installed in your environment using the command `nf-core --version`. +!!! warning "v2 parser compatibility" + + Many nf-core pipelines do not yet support the v2 syntax parser. + If you run an nf-core pipeline other than those used in this course and encounter errors, you may need to switch to the v1 parser by setting `export NXF_SYNTAX_PARSER=v1`. + See the [version notes](../info/nxf_versions.md) for details. + ## Get ready to work Once your codespace is running, there are two things you need to do before diving into the training: set your working directory for this specific course, and take a look at the materials provided. @@ -83,7 +75,10 @@ tree . -L 2 ```console . + ├── custom.config ├── greetings.csv + ├── malformed_samplesheet.csv + ├── my_params.yml ├── original-hello │ ├── hello.nf │ ├── modules @@ -102,6 +97,12 @@ We use collapsible sections like this to include expected command output in a co - **The `greetings.csv` file** is a CSV containing some minimal columnar data we use for testing purposes. +- **The `custom.config` file** is an example Nextflow configuration file used in Part 1 to demonstrate process resource overrides and `ext.args`. + +- **The `malformed_samplesheet.csv` file** is an intentionally broken samplesheet used in Part 1 to demonstrate input validation. + +- **The `my_params.yml` file** is an example params file used in Part 1 to demonstrate how to pass boolean parameters to a pipeline. + - **The `original-hello` directory** contains a copy of the source code produced by working through the complete Hello Nextflow training series (with Docker enabled). - **The `solutions` directory** contains the completed workflow scripts that result from each step of the course. @@ -113,7 +114,7 @@ Think you're ready to dive in? - [ ] I understand the goal of this course and its prerequisites - [ ] My environment is up and running -- [ ] I've made certain that the syntax parser is set to **v1** +- [ ] I'm using nf-core tools 4.0.2 (check with `nf-core --version`) - [ ] I've set my working directory appropriately If you can check all the boxes, you're good to go. diff --git a/docs/en/docs/hello_nf-core/01_run_demo.md b/docs/en/docs/hello_nf-core/01_run_demo.md index dab1298dd2..ce7edd1fdd 100644 --- a/docs/en/docs/hello_nf-core/01_run_demo.md +++ b/docs/en/docs/hello_nf-core/01_run_demo.md @@ -43,9 +43,10 @@ The `Introduction` tab provides an overview of the pipeline, including a visual ![pipeline subway map](./img/nf-core-demo-subway-cropped.png) -1. Read QC (FASTQC) -2. Adapter and quality trimming (SEQTK_TRIM) -3. Present QC for raw reads (MULTIQC) +1. Read QC ([FASTQC](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/)) +2. Adapter and quality trimming ([SEQTK_TRIM](https://github.com/lh3/seqtk)) +3. Present QC for raw reads ([MULTIQC](http://multiqc.info/)) +4. Generate a lighthearted text message from a cow ([COWPY](https://github.com/jeffbuttars/cowpy)) #### 1.1.2. Example command line @@ -80,7 +81,7 @@ nextflow pull nf-core/demo ```console Checking nf-core/demo ... - downloaded from https://github.com/nf-core/demo.git - revision: 45904cb9d1 [master] + downloaded from https://github.com/nf-core/demo.git - revision: 32893afef8 [master] ``` Nextflow does a `pull` of the pipeline code, meaning it downloads the full repository to your local drive. @@ -118,14 +119,39 @@ nextflow info nf-core/demo ??? success "Command output" ```console - project name: nf-core/demo - repository : https://github.com/nf-core/demo - local path : /workspaces/.nextflow/assets/.repos/nf-core/demo - main script : main.nf - description : An nf-core demo pipeline + project name: nf-core/demo + repository : https://github.com/nf-core/demo + local path : /workspaces/.nextflow/assets/.repos/nf-core/demo + main script : main.nf + description : An nf-core demo pipeline + revisions : + TEMPLATE + bumper + dev + fix-nxfversion + manually-merge-3_0_2 + > master (default) + nf-core-template-merge-2.13.2.dev0 + nf-core-template-merge-2.14.0 + nf-core-template-merge-2.14.1 + nf-core-template-merge-3.0.0 + nf-core-template-merge-3.0.1 + nf-core-template-merge-3.0.2 + nf-core-template-merge-3.1.0 + nf-core-template-merge-3.1.2 + nf-core-template-merge-3.2.0 + nf-core-template-merge-3.2.1 + nf-core-template-merge-3.3.1 + nf-core-template-merge-3.3.2 + nf-core-template-merge-4.0.0 + 1.0.0 [t] + 1.0.1 [t] + 1.0.2 [t] + 1.1.0 [t] + > 1.2.0 [t] ``` -!!! note +!!! info The full path may differ on your system if you're not using our training environment. @@ -178,6 +204,8 @@ tree -L 1 pipelines/nf-core/demo ├── tests ├── tower.yml └── workflows + + 7 directories, 12 files ``` As you can see, there's a lot going on in there, most of which you don't need to worry about. @@ -199,7 +227,7 @@ We won't go over the pipeline code components in this part of the course, but we You can also browse any nf-core pipeline's source code on GitHub, e.g. [github.com/nf-core/demo](https://github.com/nf-core/demo). Every nf-core pipeline follows the same directory layout, so once you know the structure, you can find configuration files, modules, and workflows for any pipeline the same way. -But for now, on to running the pipeline! +For now, on to running the pipeline! ### Takeaway @@ -217,7 +245,7 @@ Conveniently, every nf-core pipeline comes with a test profile. This is a minimal set of configuration settings for the pipeline to run using a small test dataset hosted in the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. It's a great way to quickly try out a pipeline at small scale. -!!! note +!!! tip Nextflow's configuration profile system allows you to easily switch between different container engines or execution environments. For more details, see [Hello Nextflow Part 6: Configuration](../hello_nextflow/06_hello_config.md). @@ -251,7 +279,7 @@ process { resourceLimits = [ cpus: 2, memory: '4.GB', - time: '1.h' + time: '1.h', ] } @@ -260,8 +288,7 @@ params { config_profile_description = 'Minimal test dataset to check pipeline function' // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' - + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' } ``` @@ -293,16 +320,16 @@ SAMPLE3_SE,https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/il ``` This is called a samplesheet, and is the most common form of input to nf-core pipelines. +Don't worry if you're not familiar with the data formats and types, it's not important for what follows. -!!! note - - Don't worry if you're not familiar with the data formats and types, it's not important for what follows. - -So this confirms that we have everything we need to try out the pipeline. +We now have everything we need to try out the pipeline. ### 2.2. Run the pipeline -Let's decide to use Docker for the container system and `demo-results` as the output directory, and we're ready to run the test command: +As noted above, we can use the example testing command almost as-is; we just need to specify what software packaging to use, and what to name the output directory. +Here we'll use Docker for the container system and `demo-results`, respectively. + +With that, we can run the test command: ```bash nextflow run nf-core/demo -profile docker,test --outdir demo-results @@ -311,9 +338,10 @@ nextflow run nf-core/demo -profile docker,test --outdir demo-results ??? success "Command output" ```console - N E X T F L O W ~ version 26.04.4 + N E X T F L O W ~ version 26.04.4 - Launching `https://github.com/nf-core/demo` [magical_pauling] revision: 45904cb9d1 [master] + Downloading plugin nf-schema@2.7.2 + Launching `https://github.com/nf-core/demo` [cranky_curry] revision: 32893afef8 [master] ------------------------------------------------------ @@ -322,8 +350,9 @@ nextflow run nf-core/demo -profile docker,test --outdir demo-results |\ | |__ __ / ` / \ |__) |__ } { | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/demo 1.1.0 + nf-core/demo 1.2.0 ------------------------------------------------------ + Input/output options input : https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv outdir : demo-results @@ -333,21 +362,22 @@ nextflow run nf-core/demo -profile docker,test --outdir demo-results config_profile_description: Minimal test dataset to check pipeline function Generic options - trace_report_suffix : 2025-11-21_04-57-41 + trace_report_suffix : 2026-07-03_21-31-35 Core Nextflow options revision : master - runName : magical_pauling + runName : cranky_curry containerEngine : docker launchDir : /workspaces/training/hello-nf-core workDir : /workspaces/training/hello-nf-core/work - projectDir : /workspaces/.nextflow/assets/.repos/nf-core/demo/clones/45904cb9d12db3d89900e6c479fe604ef71b297b + projectDir : /workspaces/.nextflow/assets/.repos/nf-core/demo/clones/32893afef8076a03a2767a020b3f0cab2e0b40b2 userName : root profile : docker,test - configFiles : /workspaces/.nextflow/assets/.repos/nf-core/demo/clones/45904cb9d12db3d89900e6c479fe604ef71b297b/nextflow.config + configFiles : /workspaces/.nextflow/assets/.repos/nf-core/demo/clones/32893afef8076a03a2767a020b3f0cab2e0b40b2/nextflow.config !! Only displaying parameters that differ from the pipeline defaults !! ------------------------------------------------------ + * The pipeline https://doi.org/10.5281/zenodo.12192442 @@ -357,11 +387,11 @@ nextflow run nf-core/demo -profile docker,test --outdir demo-results * Software dependencies https://github.com/nf-core/demo/blob/master/CITATIONS.md - - executor > local (7) - [ff/a6976b] NFCORE_DEMO:DEMO:FASTQC (SAMPLE3_SE) | 3 of 3 ✔ - [39/731ab7] NFCORE_DEMO:DEMO:SEQTK_TRIM (SAMPLE3_SE) | 3 of 3 ✔ - [7c/78d96e] NFCORE_DEMO:DEMO:MULTIQC | 1 of 1 ✔ + executor > local (8) + [ca/5b0f3e] NFCORE_DEMO:DEMO:FASTQC (SAMPLE3_SE) [100%] 3 of 3 ✔ + [b7/cb6812] NFCORE_DEMO:DEMO:SEQTK_TRIM (SAMPLE3_SE) [100%] 3 of 3 ✔ + [ff/6ebd98] NFCORE_DEMO:DEMO:COWPY [100%] 1 of 1 ✔ + [09/bbd1b4] NFCORE_DEMO:DEMO:MULTIQC (demo) [100%] 1 of 1 ✔ -[nf-core/demo] Pipeline completed successfully- ``` @@ -370,14 +400,14 @@ If your output matches that, congratulations! You've just run your first nf-core You'll notice that there is a lot more console output than when you run a basic Nextflow pipeline. There's a header that includes a summary of the pipeline's version, inputs and outputs, and a few elements of configuration. -!!! note +!!! info Your output will show different timestamps, execution names, and file paths, but the overall structure and process execution should be similar. Notice the line near the top of the output: ```console -Launching `https://github.com/nf-core/demo` [magical_pauling] revision: 45904cb9d1 [master] +Launching `https://github.com/nf-core/demo` [cranky_curry] revision: 32893afef8 [master] ``` This tells you which revision of the pipeline was used. @@ -385,7 +415,7 @@ Because we did not specify a version, Nextflow used the latest commit on `master For reproducible runs, you should pin a specific release using the `-r` flag: ```bash -nextflow run nf-core/demo -r 1.1.0 -profile docker,test --outdir demo-results +nextflow run nf-core/demo -r 1.2.0 -profile docker,test --outdir demo-results ``` This ensures that the same pipeline code is used every time, regardless of new commits or releases. @@ -394,14 +424,15 @@ For this training we omit `-r` for simplicity, but in production you should alwa Moving on to the execution output, let's have a look at the lines that tell us what processes were run: ```console -executor > local (7) -[ff/a6976b] NFCORE_DEMO:DEMO:FASTQC (SAMPLE3_SE) | 3 of 3 ✔ -[39/731ab7] NFCORE_DEMO:DEMO:SEQTK_TRIM (SAMPLE3_SE) | 3 of 3 ✔ -[7c/78d96e] NFCORE_DEMO:DEMO:MULTIQC | 1 of 1 ✔ +executor > local (8) +[ca/5b0f3e] NFCORE_DEMO:DEMO:FASTQC (SAMPLE3_SE) [100%] 3 of 3 ✔ +[b7/cb6812] NFCORE_DEMO:DEMO:SEQTK_TRIM (SAMPLE3_SE) [100%] 3 of 3 ✔ +[ff/6ebd98] NFCORE_DEMO:DEMO:COWPY [100%] 1 of 1 ✔ +[09/bbd1b4] NFCORE_DEMO:DEMO:MULTIQC (demo) [100%] 1 of 1 ✔ -[nf-core/demo] Pipeline completed successfully- ``` -This tells us that three processes were run, corresponding to the three tools shown in the pipeline documentation page on the nf-core website: FASTQC, SEQTK_TRIM and MULTIQC. +This tells us that four processes were run, corresponding to the four tools shown in the pipeline documentation page on the nf-core website: `FASTQC`, `SEQTK_TRIM`, `MULTIQC` and `COWPY`. The full process names as shown here, such as `NFCORE_DEMO:DEMO:MULTIQC`, are longer than what you may have seen in the introductory Hello Nextflow material. These include the names of their parent workflows and reflect the modularity of the pipeline code. @@ -419,6 +450,8 @@ tree -L 2 demo-results ```console demo-results + ├── cowpy + │ └── cowpy.txt ├── fastqc │ ├── SAMPLE1_PE │ ├── SAMPLE2_PE @@ -429,19 +462,20 @@ tree -L 2 demo-results │ └── SAMPLE3_SE ├── multiqc │ ├── multiqc_data - │ ├── multiqc_plots │ └── multiqc_report.html └── pipeline_info - ├── execution_report_2025-11-21_04-57-41.html - ├── execution_timeline_2025-11-21_04-57-41.html - ├── execution_trace_2025-11-21_04-57-41.txt + ├── execution_report_2026-07-03_21-31-35.html + ├── execution_timeline_2026-07-03_21-31-35.html + ├── execution_trace_2026-07-03_21-31-35.txt ├── nf_core_demo_software_mqc_versions.yml - ├── params_2025-11-21_04-57-46.json - └── pipeline_dag_2025-11-21_04-57-41.html + ├── params_2026-07-03_21-31-43.json + └── pipeline_dag_2026-07-03_21-31-35.html + + 12 directories, 8 files ``` That might seem like a lot. -To learn more about the `nf-core/demo` pipeline's outputs, check out its [documentation page](https://nf-co.re/demo/1.1.0/docs/output/). +To learn more about the `nf-core/demo` pipeline's outputs, check out its [documentation page](https://nf-co.re/demo/1.2.0/docs/output/). At this stage, what's important to observe is that the results are organized by module, and there is additionally a directory called `pipeline_info` containing various timestamped reports about the pipeline execution. @@ -449,7 +483,7 @@ For example, the `execution_timeline_*` file shows you what processes were run, ![execution timeline report](./img/execution_timeline.png) -!!! note +!!! info Here the tasks were not run in parallel because we are running on a minimalist machine in Github Codespaces. To see these run in parallel, try increasing the CPU allocation of your codespace and the resource limits in the test configuration. @@ -499,7 +533,7 @@ nextflow run nf-core/demo --help ```console N E X T F L O W ~ version 26.04.4 - Launching `https://github.com/nf-core/demo` [run_name] revision: 45904cb9d1 [master] + Launching `https://github.com/nf-core/demo` [adoring_meucci] revision: 32893afef8 [master] ------------------------------------------------------ @@ -508,16 +542,18 @@ nextflow run nf-core/demo --help |\ | |__ __ / ` / \ |__) |__ } { | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/demo 1.1.0 + nf-core/demo 1.2.0 ------------------------------------------------------ Typical pipeline command: nextflow run nf-core/demo -profile --input samplesheet.csv --outdir + parameter will be printed. Input/output options --input [string] Path to a metadata file containing information about the samples in the experiment. --outdir [string] The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure. + --email [string] Email address for completion summary. --multiqc_title [string] MultiQC report title. Printed as page header, used for filename if not otherwise specified. @@ -533,7 +569,7 @@ nextflow run nf-core/demo --help --help [boolean, string] Display the help message. --help_full [boolean] Display the full detailed help message. --show_hidden [boolean] Display hidden parameters in the help message (only works when --help or --help_full are provided). - !! Hiding 20 param(s), use the `--showHidden` parameter to show them !! + !! Hiding 19 param(s), use the `--showHidden` parameter to show them !! ------------------------------------------------------ * The pipeline @@ -560,29 +596,103 @@ In plain Nextflow pipelines, `--help` only works if the developer implemented it As covered in [Hello Config](../hello_nextflow/06_hello_config.md), you can set parameter values on the command line with `--param_name` or collect a set of parameters in a YAML file and pass it with `-params-file`. Both approaches work the same way with nf-core pipelines. -For example, to skip the trimming step: +For example, to skip the trimming step, we want to set the boolean parameter `skip_trim` to `true`. +A params file called `my_params.yml` is provided in your working directory with that value already set: + +```yaml title="my_params.yml" +skip_trim: true +``` + +Pass it with `-params-file`: ```bash -nextflow run nf-core/demo -profile docker,test --outdir demo-results-notrim --skip_trim +nextflow run nf-core/demo -profile docker,test --outdir demo-results-notrim -params-file my_params.yml ``` ??? success "Command output" ```console - executor > local (4) - [3f/a82c91] NFCORE_DEMO:DEMO:FASTQC (SAMPLE3_SE) | 3 of 3 ✔ - [7d/c5e014] NFCORE_DEMO:DEMO:MULTIQC | 1 of 1 ✔ + N E X T F L O W ~ version 26.04.4 + + Launching `https://github.com/nf-core/demo` [focused_heisenberg] revision: 32893afef8 [master] + + + ------------------------------------------------------ + ,--./,-. + ___ __ __ __ ___ /,-._.--~' + |\ | |__ __ / ` / \ |__) |__ } { + | \| | \__, \__/ | \ |___ \`-._,-`-, + `._,._,' + nf-core/demo 1.2.0 + ------------------------------------------------------ + + Input/output options + input : https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv + outdir : demo-results-notrim + + Process skipping options + skip_trim : true + + Institutional config options + config_profile_name : Test profile + config_profile_description: Minimal test dataset to check pipeline function + + Generic options + trace_report_suffix : 2026-07-03_22-08-47 + + Core Nextflow options + revision : master + runName : focused_heisenberg + containerEngine : docker + launchDir : /workspaces/training/hello-nf-core + workDir : /workspaces/training/hello-nf-core/work + projectDir : /workspaces/.nextflow/assets/.repos/nf-core/demo/clones/32893afef8076a03a2767a020b3f0cab2e0b40b2 + userName : root + profile : docker,test + configFiles : /workspaces/.nextflow/assets/.repos/nf-core/demo/clones/32893afef8076a03a2767a020b3f0cab2e0b40b2/nextflow.config + + !! Only displaying parameters that differ from the pipeline defaults !! + ------------------------------------------------------ + + * The pipeline + https://doi.org/10.5281/zenodo.12192442 + + * The nf-core framework + https://doi.org/10.1038/s41587-020-0439-x + + * Software dependencies + https://github.com/nf-core/demo/blob/master/CITATIONS.md + + executor > local (5) + [7a/f3599e] NFCORE_DEMO:DEMO:FASTQC (SAMPLE3_SE) [100%] 3 of 3 ✔ + [b0/2f0bdc] NFCORE_DEMO:DEMO:COWPY [100%] 1 of 1 ✔ + [c3/3c2278] NFCORE_DEMO:DEMO:MULTIQC (demo) [100%] 1 of 1 ✔ -[nf-core/demo] Pipeline completed successfully- ``` The `SEQTK_TRIM` process no longer appears in the output. -!!! info +!!! warning "Important limitations about parameter inputs" + + **Setting boolean parameters on the command line** + + Starting with Nextflow version 26.04, all values supplied on the command line are typed as strings. + For a boolean parameter like `skip_trim`, passing it as a bare flag (`--skip_trim`) or as `--skip_trim true` is evaluated as the **string** `"true"`, which fails schema validation: + + ```console + * --skip_trim (true): Value is [string] but should be [boolean] + ``` + + To set a boolean parameter to a genuine `true`/`false` value, use a `-params-file` as shown above, or set it in a config file. + String, integer and file-path parameters are unaffected and can still be set directly on the command line. + This course uses this pattern throughout for boolean parameters. + + **Using custom configuration files** Although it is technically possible to set pipeline parameters in a custom configuration file passed with `-c`, this may not override defaults already set in the pipeline's own `nextflow.config`, depending on Nextflow's configuration precedence rules. Using `--param_name` on the command line or `-params-file` is more reliable, as these always take precedence. - **As a rule of thumb:** if it appears in the `--help` output, set it via the command line or a params file rather than a config file. + As a rule of thumb: If it appears in the `--help` output, set it via the command line or a params file rather than a config file. #### 3.1.3. Parameter validation @@ -611,7 +721,7 @@ WARN: The following invalid input values have been detected: ``` The pipeline still runs, but the warning alerts you right away that `--foobar` is not a recognized parameter. -This catches typos like `--outDir` instead of `--outdir` before you waste compute time wondering why the output went to the wrong place. +This is meant to draw your attention to non-breaking typos, like `--outDir` being used instead of `--outdir`, which can help you avoid wasting time and compute. ##### 3.1.3.2. Invalid parameter values @@ -625,13 +735,14 @@ nextflow run nf-core/demo -profile docker,test --outdir demo-results --skip_trim ```console ERROR ~ Validation of pipeline parameters failed! + -- Check '.nextflow.log' file for details The following invalid input values have been detected: * --skip_trim (yes): Value is [string] but should be [boolean] ``` The pipeline stops before any processes run, saving you from a failed or incorrect execution. -Boolean parameters should be passed as flags (`--skip_trim`) without a value, or set to `true`/`false` in a params file. +As noted in section 3.1.2, boolean parameters should be set to a genuine `true`/`false` value in a params file rather than passed on the command line, since command-line values are typed as strings. #### 3.1.4. Input validation @@ -645,7 +756,7 @@ We also cover this in more detail in [Part 5: Input Validation](05_input_validat The `nf-core/demo` pipeline expects a CSV file with columns `sample`, `fastq_1`, and `fastq_2`. This is defined in a schema file (`assets/schema_input.json`) that specifies the expected structure, column types, and constraints. -??? abstract "assets/schema_input.json" +??? abstract "Schema file for inputs" ```json title="assets/schema_input.json" { @@ -686,9 +797,7 @@ This is defined in a schema file (`assets/schema_input.json`) that specifies the The schema specifies that `sample` and `fastq_1` are required, while `fastq_2` is optional (supporting both paired-end and single-end data). File paths are validated for existence and extension pattern. -##### 3.1.4.1. Create an invalid samplesheet - -Create a samplesheet with a missing column and a non-existent file path: +To demonstrate this, we provide a malformed samplesheet called `malformed_samplesheet.csv` in your working directory: ```csv title="malformed_samplesheet.csv" sample,fastq_2 @@ -696,11 +805,8 @@ SAMPLE1,/not/a/real/file.fastq.gz ``` This samplesheet is missing the required `fastq_1` column and has a non-existent file path in `fastq_2`. -Both issues will produce validation errors in the next step. - -##### 3.1.4.2. Run the demo pipeline with the invalid samplesheet -Run the demo pipeline using `malformed_samplesheet.csv` as the input. +Run the demo pipeline using `malformed_samplesheet.csv` as the input: ```bash nextflow run nf-core/demo -profile docker,test --outdir demo-results --input malformed_samplesheet.csv @@ -709,6 +815,7 @@ nextflow run nf-core/demo -profile docker,test --outdir demo-results --input mal ```console ERROR ~ Validation of pipeline parameters failed! + -- Check '.nextflow.log' file for details The following invalid input values have been detected: * --input (malformed_samplesheet.csv): Validation of file failed: @@ -740,7 +847,20 @@ ls pipelines/nf-core/demo/conf/ ``` ```console -base.config igenomes.config igenomes_ignored.config modules.config test.config test_full.config +base.config +containers_conda_lock_files_amd64.config +containers_conda_lock_files_arm64.config +containers_docker_amd64.config +containers_docker_arm64.config +containers_singularity_https_amd64.config +containers_singularity_https_arm64.config +containers_singularity_oras_amd64.config +containers_singularity_oras_arm64.config +igenomes.config +igenomes_ignored.config +modules.config +test.config +test_full.config ```
@@ -760,56 +880,18 @@ If you wish to modify any of the settings specified in these files, do not modif Instead, create your own config file and pass it with `-c`. The values you specify will override the default values set in those other files. -Let's run through a few exercises to do this in practice. +Let's try this in practice. -#### 3.2.1. Change resource allocation for a process +#### 3.2.1. Customize process resources and tool arguments -The demo pipeline assigns resources using labels defined in `base.config`. -For example, `FASTQC` uses the `process_medium` label, which allocates 6 CPUs and 36 GB of memory. +nf-core modules support two common types of configuration override: **resource allocation** (CPUs, memory, time) and **tool arguments** via `ext.args`. -The test profile caps resources via `resourceLimits`, but you can also override resources for specific processes. +Many command-line tools have arguments that are not commonly enough used to be exposed as pipeline parameters. +The `ext.args` convention lets you pass these arguments to the underlying tool through a config file instead. -Create a file called `custom.config`: +The `custom.config` file provided in your working directory demonstrates both overrides: ```groovy title="custom.config" linenums="1" -process { - withName: 'FASTQC' { - cpus = 2 - memory = 4.GB - } -} -``` - -Run the pipeline with your custom config: - -```bash -nextflow run nf-core/demo -profile docker,test --outdir demo-results-custom -c custom.config -``` - -??? success "Command output" - - ```console - executor > local (7) - [2a/f17b3e] NFCORE_DEMO:DEMO:FASTQC (SAMPLE3_SE) | 3 of 3 ✔ - [9c/e4d028] NFCORE_DEMO:DEMO:SEQTK_TRIM (SAMPLE3_SE) | 3 of 3 ✔ - [5b/a93c71] NFCORE_DEMO:DEMO:MULTIQC | 1 of 1 ✔ - -[nf-core/demo] Pipeline completed successfully- - ``` - -The `-c` flag adds your config on top of the pipeline's built-in configuration. - -#### 3.2.2. Set tool argument values with `ext.args` - -Many command-line tools have arguments that are not required and are therefore not set up as pipeline parameters unless they are very commonly used. -For those tool arguments, nf-core modules use a Nextflow convention called `ext.args` to pass arguments to the underlying tool through a configuration file. - -For example, let's add a trimming argument to the `SEQTK_TRIM` module using `ext.args`. - -##### 3.2.2.1. Update the custom configuration - -Update your `custom.config`: - -```groovy title="custom.config" linenums="1" hl_lines="6 7 8" process { withName: 'FASTQC' { cpus = 2 @@ -821,30 +903,35 @@ process { } ``` -This tells `seqtk trimfq` to trim 5 bases from the beginning of each read in addition to quality trimming. +The first block overrides `FASTQC` resource allocation. +By default, `FASTQC` uses the `process_medium` label from `base.config`, which allocates 6 CPUs and 36 GB of memory; here we cap it at 2 CPUs and 4 GB. -##### 3.2.2.2. Run the pipeline +The second block passes an extra argument to `SEQTK_TRIM` via `ext.args`. +The `-b 5` flag tells `seqtk trimfq` to trim 5 bases from the beginning of each read in addition to quality trimming. -Run the pipeline again with this config to see the effect: +Run the pipeline with this config: ```bash -nextflow run nf-core/demo -profile docker,test --outdir demo-results-extargs -c custom.config +nextflow run nf-core/demo -profile docker,test --outdir demo-results-custom -c custom.config ``` ??? success "Command output" ```console - executor > local (7) - [1e/b7a392] NFCORE_DEMO:DEMO:FASTQC (SAMPLE3_SE) | 3 of 3 ✔ - [ab/cd1234] NFCORE_DEMO:DEMO:SEQTK_TRIM (SAMPLE3_SE) | 3 of 3 ✔ - [4f/c8d105] NFCORE_DEMO:DEMO:MULTIQC | 1 of 1 ✔ + executor > local (8) + [95/b32876] NFCORE_DEMO:DEMO:FASTQC (SAMPLE1_PE) | 3 of 3 ✔ + [17/428668] NFCORE_DEMO:DEMO:SEQTK_TRIM (SAMPLE1_PE) | 3 of 3 ✔ + [cf/85991a] NFCORE_DEMO:DEMO:COWPY | 1 of 1 ✔ + [3c/94a7a0] NFCORE_DEMO:DEMO:MULTIQC (demo) | 1 of 1 ✔ -[nf-core/demo] Pipeline completed successfully- ``` -To verify the argument was applied, find the `SEQTK_TRIM` work directory hash from the run output (e.g. `work/ab/cd1234...`) and check the `.command.sh` file inside it: +The `-c` flag adds your config on top of the pipeline's built-in configuration. + +To verify the `ext.args` override took effect, find the `SEQTK_TRIM` work directory hash from the run output (e.g. `work/17/428668...`) and check the `.command.sh` file inside it: ```bash -cat work/ab/cd1234/.command.sh +cat work/17/428668/.command.sh ``` ??? success "Command output" @@ -862,30 +949,26 @@ cat work/ab/cd1234/.command.sh ... ``` -You should see `-b 5` in the `seqtk trimfq` command, confirming your `ext.args` override took effect. +You should see `-b 5` in the `seqtk trimfq` command. -##### 3.2.2.3. Overriding default values - -Some modules have `ext.args` already set by default. -For example, the `FASTQC` module is configured with `ext.args = '--quiet'` by default (defined in `conf/modules.config`). +One important thing to know about `ext.args`: if a module already has a default value set, your value will **completely replace** it rather than append to it. +For example, `FASTQC` has `ext.args = '--quiet'` set by default in `conf/modules.config`: ```groovy title="conf/modules.config" linenums="21" hl_lines="2" withName: FASTQC { - ext.args = '--quiet' + ext.args = '--quiet' publishDir = [ path: { "${params.outdir}/fastqc/${meta.id}" }, mode: params.publish_dir_mode, - pattern: "*.{html,json}" + pattern: "*.{html,json}", ] } ``` -If you provide a value for `ext.args` via a custom configuration file, that value will completely replace the default set for that process. - -So for example, if the default was `'--quiet'` and you set `ext.args = '--kmers 8'`, the `--quiet` flag will no longer be applied. +If you set `ext.args = '--kmers 8'` for `FASTQC`, the `--quiet` flag will no longer be applied. To keep both, set `ext.args = '--quiet --kmers 8'`. -This does mean you are responsible for checking what is the default configuration of tools to which you want to provide argument values with `ext.args`. +You should always check a module's default configuration before overriding `ext.args`. ### Takeaway @@ -893,4 +976,6 @@ You know how to get help from an nf-core pipeline, set parameters and understand ### What's next? -Take a break! When you're ready, move on to Part 2, where you will create your own nf-core compatible pipeline from scratch. +If you just want to run nf-core pipelines, you're done! + +If you want to learn to develop your own pipelines according to nf-core standards, take a break, and move on to Part 2 when you're ready. You will learn to create your own nf-core compatible pipeline using the nf-core template-based tools. diff --git a/docs/en/docs/hello_nf-core/02_rewrite_hello.md b/docs/en/docs/hello_nf-core/02_rewrite_hello.md index a87a7852be..8f9ea32445 100644 --- a/docs/en/docs/hello_nf-core/02_rewrite_hello.md +++ b/docs/en/docs/hello_nf-core/02_rewrite_hello.md @@ -16,10 +16,6 @@ If you're not familiar with the Hello pipeline or you could use a reminder, see - [Workflows of Workflows](../side_quests/workflows_of_workflows/index.md) - [Metadata and meta maps](../side_quests/metadata/index.md) -!!! note - - Make sure you are starting in the `hello-nf-core` directory in your terminal. - --- ## 1. Examine the pipeline code structure @@ -28,6 +24,7 @@ The nf-core project enforces strong guidelines for how pipelines are structured, Before we tackle our pipeline creation project, we need to understand that structure and organization. So let's have a look at how the pipeline code is organized in the `nf-core/demo` repository, using the `pipelines` symlink we created in Part 1. +Make sure you are starting in the `hello-nf-core` directory in your terminal. As a reminder, you can either use `tree` or use the file explorer to find and open the `nf-core/demo` directory. @@ -80,7 +77,7 @@ Here is what the relationships between the relevant code components look like: The unnamed workflow in `main.nf` is called an _entrypoint_ script. It acts as a wrapper for two kinds of nested workflows: the `DEMO` workflow containing the actual analysis logic, located in `workflows/demo.nf`, and a set of housekeeping workflows located under `subworkflows/`. The `demo.nf` workflow calls on **modules** located under `modules/`; these contain the **processes** that will perform the actual analysis steps. -!!! note +!!! info Subworkflows are not limited to housekeeping functions, and they can make use of process modules. @@ -105,7 +102,7 @@ We'll cover the relevant differences in the next part of this course, when we ta The `demo.nf` workflow calls on **modules** located under `modules/`, which we'll review next. -!!! note +!!! info Some nf-core analysis workflows display additional levels of nesting by calling on lower-level subworkflows. This is mostly used for wrapping two or more modules that are commonly used together into easily reusable pipeline segments. @@ -264,13 +261,20 @@ Once the TUI closes, you should see the following console output. | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 3.5.2 - https://nf-co.re + nf-core/tools version 4.0.2 - https://nf-co.re INFO Launching interactive nf-core pipeline creation tool. ``` -There is no explicit confirmation in the console output that the pipeline creation worked, but you should see a new directory called `core-hello`. +Once the TUI has finished, the tool reports that it created the pipeline and generated its container configuration: + +```console +INFO Creating new pipeline: 'hello' +INFO Generated container configs for the pipeline successfully. +``` + +You should now see a new directory called `core-hello`. View the contents of the new directory to see how much work you saved yourself by using the template. @@ -281,8 +285,7 @@ tree core-hello ??? abstract "Directory contents" ```console - core-hello/ - ├── README.md + core-hello ├── assets │ ├── samplesheet.csv │ └── schema_input.json @@ -292,13 +295,15 @@ tree core-hello │ ├── test.config │ └── test_full.config ├── docs - │ ├── README.md + │ ├── CONTRIBUTING.md │ ├── output.md + │ ├── README.md │ └── usage.md ├── main.nf ├── modules.json ├── nextflow.config ├── nextflow_schema.json + ├── README.md ├── subworkflows │ ├── local │ │ └── utils_nfcore_hello_pipeline @@ -318,6 +323,8 @@ tree core-hello │ │ └── tests │ │ ├── main.function.nf.test │ │ ├── main.function.nf.test.snap + │ │ ├── main.nf.test + │ │ ├── main.nf.test.snap │ │ ├── main.workflow.nf.test │ │ ├── main.workflow.nf.test.snap │ │ └── nextflow.config @@ -331,7 +338,7 @@ tree core-hello └── workflows └── hello.nf - 15 directories, 34 files + 14 directories, 37 files ``` That's a lot of files! @@ -352,9 +359,10 @@ nextflow run ./core-hello -profile docker,test --outdir core-hello-results ```console N E X T F L O W ~ version 26.04.4 - Launching `./core-hello/main.nf` [scruffy_marconi] revision: b9e9b3b8de + Launching `./core-hello/main.nf` [cheesy_avogadro] revision: d6bbba9521 - Downloading plugin nf-schema@2.5.1 + WARN: Unrecognized config option 'validation.defaultIgnoreParams' + WARN: Unrecognized config option 'validation.monochromeLogs' Input/output options input : https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv outdir : core-hello-results @@ -364,10 +372,10 @@ nextflow run ./core-hello -profile docker,test --outdir core-hello-results config_profile_description: Minimal test dataset to check pipeline function Generic options - trace_report_suffix : 2025-11-21_04-47-18 + trace_report_suffix : 2026-06-23_16-56-58 Core Nextflow options - runName : scruffy_marconi + runName : cheesy_avogadro containerEngine : docker launchDir : /workspaces/training/hello-nf-core workDir : /workspaces/training/hello-nf-core/work @@ -381,6 +389,9 @@ nextflow run ./core-hello -profile docker,test --outdir core-hello-results -[core/hello] Pipeline completed successfully- ``` +The `WARN: Unrecognized config option 'validation.*'` lines come from the version of the nf-schema plugin pinned in the freshly created template. +They are harmless and do not affect the run. + This shows you that all the basic wiring is in place. So where are the outputs? Are there any? @@ -395,12 +406,12 @@ tree core-hello-results ```console core-hello-results └── pipeline_info - ├── execution_report_2025-11-21_04-47-18.html - ├── execution_timeline_2025-11-21_04-47-18.html - ├── execution_trace_2025-11-21_04-47-18.txt + ├── execution_report_2026-06-23_16-56-58.html + ├── execution_timeline_2026-06-23_16-56-58.html + ├── execution_trace_2026-06-23_16-56-58.txt ├── hello_software_versions.yml - ├── params_2025-11-21_04-47-18.json - └── pipeline_dag_2025-11-21_04-47-18.html + ├── params_2026-06-23_16-57-00.json + └── pipeline_dag_2026-06-23_16-56-58.html 1 directory, 6 files ``` @@ -433,7 +444,7 @@ Let's take a closer look. This serves as the placeholder for our analysis workflow, with some nf-core functionality already in place. -```groovy title="core-hello/workflows/hello.nf" linenums="1" hl_lines="15 17 19 53" +```groovy title="core-hello/workflows/hello.nf" linenums="1" hl_lines="15 18 21 53" /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ IMPORT MODULES / SUBWORKFLOWS / FUNCTIONS @@ -452,14 +463,16 @@ workflow HELLO { take: ch_samplesheet // channel: samplesheet read in from --input + outdir + main: - ch_versions = channel.empty() + def ch_versions = channel.empty() // // Collate and save software versions // - def topic_versions = Channel.topic("versions") + def topic_versions = channel.topic("versions") .distinct() .branch { entry -> versions_file: entry instanceof Path @@ -476,19 +489,16 @@ workflow HELLO { "${process}:\n${tool_versions.join('\n')}" } - softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) + def ch_collated_versions = softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) .mix(topic_versions_string) .collectFile( - storeDir: "${params.outdir}/pipeline_info", + storeDir: "${outdir}/pipeline_info", name: 'hello_software_' + 'versions.yml', sort: true, newLine: true - ).set { ch_collated_versions } - - + ) emit: versions = ch_versions // channel: [ path(versions.yml) ] - } /* @@ -501,15 +511,15 @@ workflow HELLO { Compared to a basic Nextflow workflow like the one developed in [Hello Nextflow](../hello_nextflow/index.md), you'll notice a few things that are new here (highlighted lines above): - The workflow block has a name -- Workflow inputs are declared using the `take:` keyword and the channel construction is moved up to the parent workflow +- Workflow inputs are declared using the `take:` keyword (here a samplesheet channel and an output directory), and the channel construction is moved up to the parent workflow - Workflow content is placed inside a `main:` block - Outputs are declared using the `emit:` keyword These are optional features of Nextflow that make the workflow **composable**, meaning that it can be called from within another workflow. -??? note "The `Channel.topic` block" +??? note "The `channel.topic` block" - You may have noticed the `def topic_versions = Channel.topic("versions")` block starting at line 17. + You may have noticed the `def topic_versions = channel.topic("versions")` block starting at line 28. This is boilerplate housekeeping code that collects software version information from all modules automatically. nf-core is rolling out this mechanism across all pipelines in 2026, so you'll see it in all new pipelines going forward. Part 4 of this course explains how it works in detail. @@ -575,13 +585,13 @@ nextflow run original-hello/hello.nf ```console N E X T F L O W ~ version 26.04.4 - Launching `original-hello/hello.nf` [goofy_babbage] revision: e9e72441e9 + Launching `original-hello/hello.nf` [sharp_dijkstra] revision: 319b99ee58 executor > local (8) - [a4/081cec] sayHello (1) | 3 of 3 ✔ - [e7/7e9058] convertToUpper (3) | 3 of 3 ✔ - [0c/17263b] collectGreetings | 1 of 1 ✔ - [94/542280] cowpy | 1 of 1 ✔ + [23/4eb61e] sayHello (3) | 3 of 3 ✔ + [c8/81a076] convertToUpper (1) | 3 of 3 ✔ + [90/ea197e] collectGreetings | 1 of 1 ✔ + [da/3df79a] cowpy | 1 of 1 ✔ ``` If that works for you, you're ready to start hacking. @@ -702,7 +712,7 @@ While we're at it, we can also comment out the line `params.greeting = 'greeting params.character = 'turkey' ``` -!!! note +!!! info If you have the Nextflow language server extension installed, the syntax checker will light up your code with red squiggles. That's because if you put in a `take:` statement, you have to also have a `main:`. @@ -849,7 +859,7 @@ There are two important observations to make here: - The syntax for calling the imported workflow is essentially the same as the syntax for calling modules. - Everything that is related to pulling the inputs into the workflow (input parameter and channel construction) is now declared in this parent workflow. -!!! note +!!! info Naming the entrypoint workflow file `main.nf` is a convention, not a requirement. @@ -878,14 +888,14 @@ If you made all the changes correctly, this should run to completion. ```console N E X T F L O W ~ version 26.04.4 - Launching `original-hello/main.nf` [friendly_wright] revision: 1ecd2d9c0a + Launching `original-hello/main.nf` [irreverent_cajal] revision: 619249b1d7 executor > local (8) - [24/c6c0d8] HELLO:sayHello (3) | 3 of 3 ✔ - [dc/721042] HELLO:convertToUpper (3) | 3 of 3 ✔ - [48/5ab2df] HELLO:collectGreetings | 1 of 1 ✔ - [e3/693b7e] HELLO:cowpy | 1 of 1 ✔ - Output: /workspaces/training/hello-nf-core/work/e3/693b7e48dc119d0c54543e0634c2e7/cowpy-COLLECTED-test-batch-output.txt + [50/b02a90] HELLO:sayHello (1) | 3 of 3 ✔ + [c0/3c336a] HELLO:convertToUpper (2) | 3 of 3 ✔ + [5c/47bb4f] HELLO:collectGreetings | 1 of 1 ✔ + [07/bfc706] HELLO:cowpy | 1 of 1 ✔ + Output: /workspaces/training/hello-nf-core/work/07/bfc7061fa521e86f4e1954191ab4c4/cowpy-COLLECTED-test-batch-output.txt ``` This means we've successfully upgraded our HELLO workflow to be composable. @@ -930,14 +940,16 @@ workflow HELLO { take: ch_samplesheet // channel: samplesheet read in from --input + outdir + main: - ch_versions = channel.empty() + def ch_versions = channel.empty() // // Collate and save software versions // - def topic_versions = Channel.topic("versions") + def topic_versions = channel.topic("versions") .distinct() .branch { entry -> versions_file: entry instanceof Path @@ -954,19 +966,16 @@ workflow HELLO { "${process}:\n${tool_versions.join('\n')}" } - softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) + def ch_collated_versions = softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) .mix(topic_versions_string) .collectFile( - storeDir: "${params.outdir}/pipeline_info", + storeDir: "${outdir}/pipeline_info", name: 'hello_software_' + 'versions.yml', sort: true, newLine: true - ).set { ch_collated_versions } - - + ) emit: versions = ch_versions // channel: [ path(versions.yml) ] - } /* @@ -976,8 +985,8 @@ workflow HELLO { */ ``` -The highlighted lines define the composable workflow structure: `workflow HELLO {`, `take:`, `main:`, and `emit:`. -The large block between lines 17–34 is more substantial: it handles software version capture using topic channels, a mechanism nf-core is rolling out across all pipelines in 2026. +This is the composable workflow structure: a named `workflow HELLO {` block with `take:`, `main:`, and `emit:`. +The block under `// Collate and save software versions` is more substantial: it handles software version capture using topic channels, a mechanism nf-core is rolling out across all pipelines in 2026. We'll explain it in Part 4; for now, treat it as boilerplate that you can leave untouched. We need to add the relevant code from the composable version of the original workflow that we developed in section 2. @@ -989,7 +998,7 @@ We're going to tackle this in the following stages: 3. Add the workflow logic to the `main` block 4. Update the `emit` block -!!! note +!!! info We're going to ignore the version capture block for this first pass. Part 4 explains how it works. @@ -1077,9 +1086,10 @@ Two more interesting observations here: The nf-core project has a lot of prebuilt functionality around the concept of the samplesheet, which is typically a CSV file containing columnar data. Since that is essentially what our `greetings.csv` file is, we'll keep the current `take` declaration as is, and simply update the name of the input channel in the next step. -```groovy title="core-hello/workflows/hello.nf" linenums="21" +```groovy title="core-hello/workflows/hello.nf" linenums="17" take: ch_samplesheet // channel: samplesheet read in from --input + outdir ``` The input handling will be done upstream of this workflow (not in this code file). @@ -1109,20 +1119,21 @@ As a reminder, this is the relevant code in the original workflow, which didn't We need to copy the code that comes after `main:` into the new version of the workflow. There is already some code in there that has to do with capturing the versions of the tools that get run by the workflow. We're going to leave that alone for now (we'll deal with the tool versions later). -We'll keep the `ch_versions = channel.empty()` initialization at the top, then insert our workflow logic, keeping the version collation code at the end. +We'll keep the `def ch_versions = channel.empty()` initialization at the top, then insert our workflow logic, keeping the version collation code at the end. This ordering makes sense because in a real pipeline, the processes would emit version information that would be added to the `ch_versions` channel as the workflow runs. === "After" - ```groovy title="core-hello/workflows/hello.nf" linenums="19" hl_lines="10-20" + ```groovy title="core-hello/workflows/hello.nf" linenums="15" hl_lines="11-21" workflow HELLO { take: ch_samplesheet // channel: samplesheet read in from --input + outdir main: - ch_versions = channel.empty() + def ch_versions = channel.empty() // emit a greeting sayHello(greeting_ch) @@ -1139,7 +1150,7 @@ This ordering makes sense because in a real pipeline, the processes would emit v // // Collate and save software versions // - def topic_versions = Channel.topic("versions") + def topic_versions = channel.topic("versions") .distinct() .branch { entry -> versions_file: entry instanceof Path @@ -1156,37 +1167,36 @@ This ordering makes sense because in a real pipeline, the processes would emit v "${process}:\n${tool_versions.join('\n')}" } - softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) + def ch_collated_versions = softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) .mix(topic_versions_string) .collectFile( - storeDir: "${params.outdir}/pipeline_info", + storeDir: "${outdir}/pipeline_info", name: 'hello_software_' + 'versions.yml', sort: true, newLine: true - ).set { ch_collated_versions } - - + ) emit: versions = ch_versions // channel: [ path(versions.yml) ] - } ``` === "Before" - ```groovy title="core-hello/workflows/hello.nf" linenums="19" + ```groovy title="core-hello/workflows/hello.nf" linenums="15" workflow HELLO { take: ch_samplesheet // channel: samplesheet read in from --input + outdir + main: - ch_versions = channel.empty() + def ch_versions = channel.empty() // // Collate and save software versions // - def topic_versions = Channel.topic("versions") + def topic_versions = channel.topic("versions") .distinct() .branch { entry -> versions_file: entry instanceof Path @@ -1203,36 +1213,31 @@ This ordering makes sense because in a real pipeline, the processes would emit v "${process}:\n${tool_versions.join('\n')}" } - softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) + def ch_collated_versions = softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) .mix(topic_versions_string) .collectFile( - storeDir: "${params.outdir}/pipeline_info", + storeDir: "${outdir}/pipeline_info", name: 'hello_software_' + 'versions.yml', sort: true, newLine: true - ).set { ch_collated_versions } - - + ) emit: versions = ch_versions // channel: [ path(versions.yml) ] - } ``` -You'll notice that we also added a blank line before `main:` to make the code more readable. - This looks great, but we still need to update the name of the channel we're passing to the `sayHello()` process from `greeting_ch` to `ch_samplesheet` as shown below, to match what is written under the `take:` keyword. === "After" - ```groovy title="core-hello/workflows/hello.nf" linenums="26" + ```groovy title="core-hello/workflows/hello.nf" linenums="25" // emit a greeting (updated to use the nf-core convention for samplesheets) sayHello(ch_samplesheet) ``` === "Before" - ```groovy title="core-hello/workflows/hello.nf" linenums="26" + ```groovy title="core-hello/workflows/hello.nf" linenums="25" // emit a greeting sayHello(greeting_ch) ``` @@ -1245,7 +1250,7 @@ Finally, we need to update the `emit` block to include the declaration of the wo === "After" - ```groovy title="core-hello/workflows/hello.nf" linenums="69" hl_lines="2" + ```groovy title="core-hello/workflows/hello.nf" linenums="71" hl_lines="2" emit: cowpy_hellos = cowpy.out versions = ch_versions // channel: [ path(versions.yml) ] @@ -1253,7 +1258,7 @@ Finally, we need to update the `emit` block to include the declaration of the wo === "Before" - ```groovy title="core-hello/workflows/hello.nf" linenums="69" + ```groovy title="core-hello/workflows/hello.nf" linenums="71" emit: versions = ch_versions // channel: [ path(versions.yml) ] ``` @@ -1322,7 +1327,8 @@ workflow CORE_HELLO { // WORKFLOW: Run pipeline // HELLO ( - samplesheet + samplesheet, + params.outdir, ) } /* @@ -1359,7 +1365,6 @@ workflow { // SUBWORKFLOW: Run completion tasks // PIPELINE_COMPLETION ( - params.outdir, params.monochrome_logs, ) } @@ -1422,7 +1427,7 @@ If we open up that file and scroll down, we come to this chunk of code: This is the channel factory that parses the samplesheet and passes it on in a form that is ready to be consumed by the HELLO workflow. -!!! note +!!! info The syntax above is a little different from what we've used previously, but basically this: @@ -1593,13 +1598,53 @@ And while we're at it, let's tighten the default resource limits to ensure this This completes the code modifications we need to do. -### 5.4. Run the pipeline with the test profile +### 5.4. Disable parameter validation + +We replaced the templated samplesheet parsing with our own simple channel construction, but the template still ships a `nextflow_schema.json` and `assets/schema_input.json` describing a fastq-based samplesheet. +Because we have not yet adapted those schemas to our `greetings.csv` format, we need to switch off parameter validation for now (we'll set it up properly later). + +Open `core-hello/nextflow.config` and set `validate_params` to `false`: + +=== "After" + + ```groovy title="core-hello/nextflow.config" linenums="37" + validate_params = false + ``` + +=== "Before" + + ```groovy title="core-hello/nextflow.config" linenums="37" + validate_params = true + ``` + +We set this in the config file rather than on the command line because starting in Nextflow version 26.04, all values supplied on the command line are typed as strings. +As a result, Boolean parameters must be set in a config file or a `-params-file` to take a genuine `true`/`false` value. + +For example, using `--validate_params false` here would evaluate as the **string** `"false"`, which leaves validation switched on. + +!!! tip "v2 parser compatibility lines in `nextflow.config`" + + Speaking of v2 syntax, you may notice these two lines just below the `params` block in the config file: + + ```groovy + outputDir = params.outdir + workflow.output.mode = params.publish_dir_mode + ``` + + These are required for compatibility with the v2 syntax parser. + + - With the v2 syntax, `params.*` variables cannot be referenced directly inside `publishDir` directives in process modules, so `outputDir` is defined here as a top-level config variable that those directives can access. + + - `workflow.output.mode` sets the default publishing mode for the v2 workflow output block. + + Both are generated automatically by the nf-core pipeline template and do not need to be modified. + +### 5.5. Run the pipeline with the test profile That was a lot, but we can finally try running the pipeline! -Note that we have to add `--validate_params false` to the command line because we didn't set up the validation yet (that will come later). ```bash -nextflow run core-hello --outdir core-hello-results -profile test,docker --validate_params false +nextflow run core-hello --outdir core-hello-results -profile test,docker ``` If you've done all of the modifications correctly, it should run to completion. @@ -1609,7 +1654,7 @@ If you've done all of the modifications correctly, it should run to completion. ```console N E X T F L O W ~ version 26.04.4 - Launching `core-hello/main.nf` [condescending_allen] revision: b9e9b3b8de + Launching `core-hello/main.nf` [voluminous_caravaggio] revision: d6bbba9521 Input/output options input : /workspaces/training/hello-nf-core/core-hello/assets/greetings.csv @@ -1621,10 +1666,10 @@ If you've done all of the modifications correctly, it should run to completion. Generic options validate_params : false - trace_report_suffix : 2025-11-21_07-29-37 + trace_report_suffix : 2026-06-23_16-58-45 Core Nextflow options - runName : condescending_allen + runName : voluminous_caravaggio containerEngine : docker launchDir : /workspaces/training/hello-nf-core workDir : /workspaces/training/hello-nf-core/work @@ -1635,17 +1680,17 @@ If you've done all of the modifications correctly, it should run to completion. !! Only displaying parameters that differ from the pipeline defaults !! ------------------------------------------------------ - executor > local (1) - [ed/727b7e] CORE_HELLO:HELLO:sayHello (3) | 3 of 3 ✔ - [45/bb6096] CORE_HELLO:HELLO:convertToUpper (3) | 3 of 3 ✔ - [81/7e2e34] CORE_HELLO:HELLO:collectGreetings | 1 of 1 ✔ - [96/9442a1] CORE_HELLO:HELLO:cowpy | 1 of 1 ✔ + executor > local (8) + [30/fc3bdb] CORE_HELLO:HELLO:sayHello (1) | 3 of 3 ✔ + [55/58b611] CORE_HELLO:HELLO:convertToUpper (1) | 3 of 3 ✔ + [12/83c0bc] CORE_HELLO:HELLO:collectGreetings | 1 of 1 ✔ + [18/4894fd] CORE_HELLO:HELLO:cowpy | 1 of 1 ✔ -[core/hello] Pipeline completed successfully- ``` As you can see, this produced the typical nf-core summary at the start thanks to the initialisation subworkflow, and the lines for each module now show the full PIPELINE:WORKFLOW:module names. -### 5.5. Find the pipeline outputs +### 5.6. Find the pipeline outputs The question now is: where are the outputs of the pipeline? And the answer is quite interesting: there are now two different places to look for the results. @@ -1661,17 +1706,17 @@ tree core-hello-results ```console core-hello-results └── pipeline_info - ├── execution_report_2025-11-21_04-47-18.html - ├── execution_report_2025-11-21_07-29-37.html - ├── execution_timeline_2025-11-21_04-47-18.html - ├── execution_timeline_2025-11-21_07-29-37.html - ├── execution_trace_2025-11-21_04-47-18.txt - ├── execution_trace_2025-11-21_07-29-37.txt + ├── execution_report_2026-06-23_16-56-58.html + ├── execution_report_2026-06-23_16-58-45.html + ├── execution_timeline_2026-06-23_16-56-58.html + ├── execution_timeline_2026-06-23_16-58-45.html + ├── execution_trace_2026-06-23_16-56-58.txt + ├── execution_trace_2026-06-23_16-58-45.txt ├── hello_software_versions.yml - ├── params_2025-11-21_04-47-13.json - ├── params_2025-11-21_07-29-41.json - ├── pipeline_dag_2025-11-21_04-47-18.html - └── pipeline_dag_2025-11-21_07-29-37.html + ├── params_2026-06-23_16-57-00.json + ├── params_2026-06-23_16-58-47.json + ├── pipeline_dag_2026-06-23_16-56-58.html + └── pipeline_dag_2026-06-23_16-58-45.html 1 directory, 12 files ``` @@ -1681,7 +1726,7 @@ This time you see all the tasks that were run as expected. ![execution timeline report for the Hello pipeline](./img/execution_timeline_hello.png) -!!! note +!!! info Once again the tasks were not run in parallel because we are running on a minimalist machine in Github Codespaces. To see these run in parallel, try increasing the CPU allocation of your codespace and the resource limits in the test configuration. diff --git a/docs/en/docs/hello_nf-core/03_use_module.md b/docs/en/docs/hello_nf-core/03_use_module.md index 39d4602e69..51442d2c44 100644 --- a/docs/en/docs/hello_nf-core/03_use_module.md +++ b/docs/en/docs/hello_nf-core/03_use_module.md @@ -23,7 +23,7 @@ To demonstrate how this works, we'll replace the custom `collectGreetings` modul You can test that it runs successfully by running the following command: ```bash - nextflow run . --outdir core-hello-results -profile test,docker --validate_params false + nextflow run . --outdir core-hello-results -profile test,docker ``` --- @@ -48,7 +48,7 @@ Navigate to the modules page in your web browser and use the search bar to searc As you can see, there are quite a few results, many of them modules designed to concatenate very specific types of files. Among them, you should see one called `find_concatenate` that is general-purpose. -!!! note "Module naming convention" +!!! info "Module naming convention" The underscore (`_`) is used as a stand-in for the slash (`/`) character in module names. @@ -118,7 +118,8 @@ This displays documentation about the module, including its inputs, outputs, and | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 3.5.2 - https://nf-co.re + nf-core/tools version 4.0.2 - https://nf-co.re + ╭─ Module: find/concatenate ──────────────────────────────────────────────────╮ @@ -191,15 +192,13 @@ Now that we've found the module we want, we need to add it to our pipeline's sou The good news is that the nf-core project includes some tooling to make this part easy. Specifically, the `nf-core modules install` command makes it possible to automate retrieving the code and making it available to your project in a single step. -Navigate to your pipeline directory and run the installation command: +Make sure your working directory is the root of the `core-hello` pipeline project, then run the installation command: ```bash cd core-hello nf-core modules install find/concatenate ``` -The tool will proceed to install the module. - ??? success "Command output" ```console @@ -210,26 +209,20 @@ The tool will proceed to install the module. | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 3.5.2 - https://nf-co.re + nf-core/tools version 4.0.2 - https://nf-co.re INFO Installing 'find/concatenate' - INFO Use the following statement to include this module: - - include { FIND_CONCATENATE } from '../modules/nf-core/find/concatenate/main' + NotADirectoryError: [Errno 20] Not a directory: + 'modules/local/cowpy.nf/meta.yml' ``` -The command automatically: - -- Downloads the module files to `modules/nf-core/find/concatenate/` -- Updates `modules.json` to track the installed module -- Provides you with the correct `include` statement to use in your workflow +The command downloads the module files to `modules/nf-core/find/concatenate/` and updates `modules.json` to track the installed module. +You can ignore the `NotADirectoryError` at the end; it happens because nf-core/tools 4.0.2 expects every local module to live in its own directory (`modules/local//main.nf`), while `core-hello` still uses single-file local modules at this stage. +However, the `find/concatenate` module is installed correctly, and `modules.json` is updated as expected. +We'll convert `cowpy` to the directory layout in Part 4. -!!! tip - - Always make sure your current working directory is the root of your pipeline project before running the module installation command. - -Let's check that the module was installed correctly: +Let's check that the module files are in place: ```bash tree -L 4 modules @@ -255,35 +248,67 @@ tree -L 4 modules 5 directories, 7 files ``` -You can also verify the installation by asking the nf-core utility to list locally installed modules: - -```bash -nf-core modules list local -``` - -??? success "Command output" - - ```console - INFO Repository type: pipeline - INFO Modules installed in '.': - - ┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ - ┃ Module Name ┃ Repository ┃ Version SHA ┃ Message ┃ Date ┃ - ┡━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ - │ find/concaten… │ nf-core/modules │ 6d46786 │ Support for │ 2026-04-23 │ - │ │ │ │ apptainer as │ │ - │ │ │ │ well as │ │ - │ │ │ │ singularity │ │ - │ │ │ │ for .sif in │ │ - │ │ │ │ `container` │ │ - │ │ │ │ (#11260) │ │ - └────────────────┴─────────────────┴─────────────┴────────────────┴────────────┘ +You can also confirm the installation by inspecting `modules.json`, which now lists `find/concatenate` under the nf-core/modules repository. + +??? abstract File contents + + ```json title="modules.json" + { + "name": "core/hello", + "homePage": "https://github.com/core/hello", + "repos": { + "https://github.com/nf-core/modules.git": { + "modules": { + "nf-core": { + "find/concatenate": { + "branch": "master", + "git_sha": "6d46786420b4d7bc88eba026eb389c0c5535d120", + "installed_by": [ + "modules" + ] + } + } + }, + "subworkflows": { + "nf-core": { + "utils_nextflow_pipeline": { + "branch": "master", + "git_sha": "05954dab2ff481bcb999f24455da29a5828af08d", + "installed_by": [ + "subworkflows" + ] + }, + "utils_nfcore_pipeline": { + "branch": "master", + "git_sha": "a3fb7351b1fdb2b1de282b765816bbea190e86a8", + "installed_by": [ + "subworkflows" + ] + }, + "utils_nfschema_plugin": { + "branch": "master", + "git_sha": "fdc08b8b1ae74f56686ce21f7ea11ad11990ce57", + "installed_by": [ + "subworkflows" + ] + } + } + } + } + } + } ``` This confirms that the `find/concatenate` module is now part of your project's source code. - However, to actually use the new module, we need to import it into our pipeline. +!!! info "Checking installed modules with `nf-core modules list local`" + + The nf-core tools provide a command to list all modules currently installed in a pipeline: `nf-core modules list local`. + Under normal circumstances this is a convenient way to verify an installation. + However, in nf-core/tools 4.0.2, this command returns an empty table when the pipeline contains any single-file local modules (such as `modules/local/cowpy.nf`), because the tool only recognizes the newer directory layout (`modules/local//main.nf`). + Since `core-hello` still uses single-file local modules at this stage, `modules.json` and the `modules/nf-core/` directory are the reliable checks — as shown above. + ### 1.5. Update the module imports Let's replace the `include` statement for the `collectGreetings` module with the one for `FIND_CONCATENATE` in the imports section of the `workflows/hello.nf` workflow. @@ -343,7 +368,7 @@ At this point, you might be tempted to dive in and start editing code, but it's We're going to tackle that as a separate section because it involves a new mechanism we haven't covered yet: metadata maps. -!!! note +!!! info You can optionally delete the `collectGreetings.nf` file: @@ -371,7 +396,7 @@ This will allow us to determine whether we can just treat the new module as a dr Ideally this is something you should do _before_ you even install the module, but hey, better late than never. (For what it's worth, there is an `uninstall` command to get rid of modules you decide you no longer want.) -!!! note +!!! info The FIND_CONCATENATE process includes some rather clever handling of different compression types, file extensions and so on that aren't strictly relevant to what we're trying to show you here, so we'll ignore most of it and focus only on the parts that are important. @@ -510,7 +535,7 @@ As mentioned earlier, the `tuple val(meta), path(files_in)` input setup is a sta Hopefully you can start to see how useful this can be. Not only does it allow you to name outputs based on metadata, but you can also do things like use it to apply different parameter values, and in combination with specific operators, you can even group, sort or filter out data as it flows through the pipeline. -!!! note "Learn more about metadata" +!!! info "Learn more about metadata" For a comprehensive introduction to working with metadata in Nextflow workflows, including how to read metadata from samplesheets and use it to customize processing, see the [Metadata in workflows](../side_quests/metadata/index.md) side quest. @@ -541,7 +566,7 @@ Now that you know everything about metamaps (or enough for the purposes of this For the sake of clarity, we'll break this down and cover each step separately. -!!! note +!!! info All the changes shown below are made to the workflow logic in the `main` block in the `core-hello/workflows/hello.nf` workflow file. @@ -751,7 +776,7 @@ The `#!groovy .map { meta, file -> file }` operation extracts the file from the Then it's just a matter of passing `ch_for_cowpy` to `cowpy` instead of `collectGreetings.out.outfile` in that last line. -!!! note +!!! info In the next part of the course, we'll update `cowpy` to work with metadata tuples directly, so this extraction step will no longer be necessary. @@ -760,7 +785,7 @@ Then it's just a matter of passing `ch_for_cowpy` to `cowpy` instead of `collect Let's test that the workflow works with the newly integrated `find/concatenate` module: ```bash -nextflow run . --outdir core-hello-results -profile test,docker --validate_params false +nextflow run . --outdir core-hello-results -profile test,docker ``` This should run reasonably quickly. @@ -768,40 +793,40 @@ This should run reasonably quickly. ??? success "Command output" ```console - N E X T F L O W ~ version 26.04.4 - - Launching `./main.nf` [evil_pike] revision: b9e9b3b8de - - Input/output options - input : /workspaces/training/hello-nf-core/core-hello/assets/greetings.csv - outdir : core-hello-results - - Institutional config options - config_profile_name : Test profile - config_profile_description: Minimal test dataset to check pipeline function - - Generic options - validate_params : false - trace_report_suffix : 2025-10-30_18-50-58 - - Core Nextflow options - runName : evil_pike - containerEngine : docker - launchDir : /workspaces/training/hello-nf-core/core-hello - workDir : /workspaces/training/hello-nf-core/core-hello/work - projectDir : /workspaces/training/hello-nf-core/core-hello - userName : root - profile : test,docker - configFiles : /workspaces/training/hello-nf-core/core-hello/nextflow.config - - !! Only displaying parameters that differ from the pipeline defaults !! - ------------------------------------------------------ - executor > local (8) - [b3/f005fd] CORE_HELLO:HELLO:sayHello (3) | 3 of 3 ✔ - [08/f923d0] CORE_HELLO:HELLO:convertToUpper (3) | 3 of 3 ✔ - [34/3729a9] CORE_HELLO:HELLO:FIND_CONCATENATE (test) | 1 of 1 ✔ - [24/df918a] CORE_HELLO:HELLO:cowpy | 1 of 1 ✔ - -[core/hello] Pipeline completed successfully- + N E X T F L O W ~ version 26.04.4 + + Launching `./main.nf` [cheesy_bhabha] revision: d6bbba9521 + + Input/output options + input : /workspaces/training/hello-nf-core/core-hello/assets/greetings.csv + outdir : core-hello-results + + Institutional config options + config_profile_name : Test profile + config_profile_description: Minimal test dataset to check pipeline function + + Generic options + validate_params : false + trace_report_suffix : 2026-06-23_16-55-02 + + Core Nextflow options + runName : cheesy_bhabha + containerEngine : docker + launchDir : /workspaces/training/hello-nf-core/core-hello + workDir : /workspaces/training/hello-nf-core/core-hello/work + projectDir : /workspaces/training/hello-nf-core/core-hello + userName : root + profile : test,docker + configFiles : /workspaces/training/hello-nf-core/core-hello/nextflow.config + + !! Only displaying parameters that differ from the pipeline defaults !! + ------------------------------------------------------ + executor > local (8) + [bf/aa86d7] CORE_HELLO:HELLO:sayHello (3) | 3 of 3 ✔ + [0a/df448e] CORE_HELLO:HELLO:convertToUpper (3) | 3 of 3 ✔ + [82/ded72f] CORE_HELLO:HELLO:FIND_CONCATENATE (test) | 1 of 1 ✔ + [9d/0130bf] CORE_HELLO:HELLO:cowpy | 1 of 1 ✔ + -[core/hello] Pipeline completed successfully- ``` Notice that `FIND_CONCATENATE` now appears in the process execution list instead of `collectGreetings`. diff --git a/docs/en/docs/hello_nf-core/04_make_module.md b/docs/en/docs/hello_nf-core/04_make_module.md index aa4ae1192b..950bd09ef5 100644 --- a/docs/en/docs/hello_nf-core/04_make_module.md +++ b/docs/en/docs/hello_nf-core/04_make_module.md @@ -22,7 +22,7 @@ After that, we'll show you how to use the template-based module creation to work You can test that it runs successfully by running the following command: ```bash - nextflow run . --outdir core-hello-results -profile test,docker --validate_params false + nextflow run . --outdir core-hello-results -profile test,docker ``` --- @@ -245,7 +245,7 @@ Be sure to make **both** changes, otherwise you will get an error when you run t Let's run the workflow to test that everything is working correctly after these changes. ```bash -nextflow run . --outdir core-hello-results -profile test,docker --validate_params false +nextflow run . --outdir core-hello-results -profile test,docker ``` ??? success "Command output" @@ -393,7 +393,7 @@ This is technically not required, but it's good practice to refer to named outpu Let's run the workflow to test that everything is working correctly after these changes. ```bash -nextflow run . --outdir core-hello-results -profile test,docker --validate_params false +nextflow run . --outdir core-hello-results -profile test,docker ``` ??? success "Command output" @@ -403,7 +403,7 @@ nextflow run . --outdir core-hello-results -profile test,docker --validate_param Launching `./main.nf` [modest_saha] revision: b9e9b3b8de - Downloading plugin nf-schema@2.5.1 + Downloading plugin nf-schema@2.7.2 Input/output options input : /workspaces/training/hello-nf-core/core-hello/assets/greetings.csv outdir : core-hello-results @@ -531,7 +531,7 @@ You can see we made three changes. As a result, the module interface is now simpler: it only expects the essential metadata and file inputs. -!!! note +!!! info The `?:` operator is often called the 'Elvis operator' because it looks like a sideways Elvis Presley face, with the `?` character symbolizing the wave in his hair. @@ -621,7 +621,7 @@ Let's test that the workflow still works as expected, specifying a different cha Run this command using `kosh`, one of the more... enigmatic options: ```bash -nextflow run . --outdir core-hello-results -profile test,docker --validate_params false --character kosh +nextflow run . --outdir core-hello-results -profile test,docker --character kosh ``` ??? success "Command output" @@ -675,16 +675,16 @@ cat work/38/eb29ea*/cowpy-test.txt ??? success "Command output" ```console - _________ - / HELLO \ - | HOLA | - \ BONJOUR / - --------- - \ + _________ + / BONJOUR \ + | HELLO | + \ HOLA / + --------- \ + \ \ ___ _____ ___ - / \ / /| / \ + / \ / /| / \ | | / / | | | | | /____/ | | | | | | | | | | @@ -729,7 +729,7 @@ To summarize the benefits of this approach: - **Portability**: Modules can be reused without hardcoded tool options - **No workflow changes**: Adding or changing tool options doesn't require updating workflow code -!!! note +!!! info The `ext.args` system has powerful additional capabilities not covered here, including switching argument values dynamically based on metadata. See the [nf-core module specifications](https://nf-co.re/docs/guidelines/components/modules) for more details. @@ -839,7 +839,7 @@ In case you're wondering, the `ext.prefix` closure has access to the correct pie Let's test that the workflow still works as expected. ```bash -nextflow run . --outdir core-hello-results -profile test,docker --validate_params false +nextflow run . --outdir core-hello-results -profile test,docker ``` ??? success "Command output" @@ -976,7 +976,7 @@ That's it! Let's have a look at what happens if we run the pipeline now. ```bash -nextflow run . --outdir core-hello-results -profile test,docker --validate_params false +nextflow run . --outdir core-hello-results -profile test,docker ``` ??? success "Command output" @@ -1055,7 +1055,7 @@ Now the `core-hello-results` also contains the outputs of the `COWPY` module. You can see that Nextflow created this hierarchy of directories based on the names of the workflow and of the module. -!!! note +!!! info You may notice `hello_software_versions.yml` in `pipeline_info/`. It currently only contains version information from `FIND_CONCATENATE`, because `COWPY` doesn't report its version yet. @@ -1160,7 +1160,7 @@ No changes to the script block are needed — the version is declared statically #### 1.6.2. Run the pipeline and inspect the versions report ```bash -nextflow run . --outdir core-hello-results -profile test,docker --validate_params false +nextflow run . --outdir core-hello-results -profile test,docker ``` ??? success "Command output" @@ -1189,11 +1189,14 @@ FIND_CONCATENATE: coreutils: 9.4 find: 4.6.0 pigz: 2.8 +Workflow: + core/hello: v1.0.0dev + Nextflow: 26.04.4 ``` The workflow-side collection — the `Channel.topic("versions")` block you saw in the placeholder workflow in Part 2 — subscribes to the topic and writes this combined report automatically. -!!! note "Backwards compatibility" +!!! info "Backwards compatibility" The `versions_file` branch in the workflow's topic channel block exists to handle modules that haven't yet been updated to use `topic: versions` and still write a `versions.yml` file in the script block with `emit: versions`. Both styles are supported simultaneously during the transition. @@ -1283,16 +1286,16 @@ process COWPY { label 'process_single' conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + container "${ workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE': - 'biocontainers/YOUR-TOOL-HERE' }" + 'quay.io/biocontainers/YOUR-TOOL-HERE' }" input: tuple val(meta), path(input) // Pattern 1: Metadata tuples ✓ output: tuple val(meta), path("*"), emit: output - tuple val("${task.process}"), val('cowpy'), val("1.1.5"), topic: versions, emit: versions_cowpy + tuple val("${task.process}"), val('cowpy'), eval("cowpy --version"), topic: versions, emit: versions_cowpy when: task.ext.when == null || task.ext.when @@ -1367,9 +1370,9 @@ The default code offers to toggle between Docker and Singularity, but we're goin label 'process_single' conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + container "${ workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE': - 'biocontainers/YOUR-TOOL-HERE' }" + 'quay.io/biocontainers/YOUR-TOOL-HERE' }" ``` #### 2.2.2. Conda environment @@ -1437,13 +1440,13 @@ Update the input and output blocks: === "Before" - ```groovy title="modules/local/cowpy/main.nf" linenums="8" hl_lines="2 5" + ```groovy title="modules/local/cowpy/main.nf" linenums="8" hl_lines="2 5 6" input: tuple val(meta), path(input) output: tuple val(meta), path("*"), emit: output - tuple val("${task.process}"), val('cowpy'), val("1.1.5"), topic: versions , emit: versions_cowpy + tuple val("${task.process}"), val('cowpy'), eval("cowpy --version"), topic: versions , emit: versions_cowpy ``` This specifies: @@ -1451,6 +1454,7 @@ This specifies: - The input file parameter name (`input_file` instead of generic `input`) - The output filename using the configurable prefix pattern (`#!groovy ${prefix}.txt` instead of wildcard `*`) - A descriptive emit name (`cowpy_output` instead of generic `output`) +- A static version string (`#!groovy val("1.1.5")`) in place of the template's `#!groovy eval("cowpy --version")`, matching the manual module from section 1.6 (the `cowpy` tool does not expose a `--version` flag) If you're using the Nextflow language server to validate syntax, the `#!groovy ${prefix}` part will be flagged as an error at this stage because we haven't added it to the script block yet. Let's get to that now. @@ -1574,7 +1578,7 @@ All we need to do to try out this new version of the `COWPY` module is to switch Let's run the pipeline to test it. ```bash -nextflow run . --outdir core-hello-results -profile test,docker --validate_params false +nextflow run . --outdir core-hello-results -profile test,docker ``` ??? success "Command output" diff --git a/docs/en/docs/hello_nf-core/05_input_validation.md b/docs/en/docs/hello_nf-core/05_input_validation.md index 97ad5bef2b..5b97798616 100644 --- a/docs/en/docs/hello_nf-core/05_input_validation.md +++ b/docs/en/docs/hello_nf-core/05_input_validation.md @@ -18,7 +18,7 @@ In this fifth part of the Hello nf-core training course, we show you how to use You can test that it runs successfully by running the following command: ```bash - nextflow run . --outdir core-hello-results -profile test,docker --validate_params false + nextflow run . --outdir core-hello-results -profile test,docker ``` --- @@ -82,7 +82,7 @@ nf-schema is the successor to the deprecated nf-validation plugin and uses stand ```groovy plugins { - id 'nf-schema@2.1.1' + id 'nf-schema@2.7.2' } ``` @@ -140,23 +140,40 @@ Now let's apply these principles in practice, starting with parameter validation Let's start by adding parameter validation to our pipeline. This validates command-line flags like `--input`, `--outdir`, and `--batch`. -### 1.1. Configure validation to skip input file validation +### 1.1. Enable validation and skip input file validation The nf-core pipeline template comes with nf-schema already installed and configured: - The nf-schema plugin is installed via the `plugins{}` block in `nextflow.config` -- Parameter validation is enabled by default via `params.validate_params = true` +- Parameter validation is controlled by `params.validate_params` - The validation is performed by the `UTILS_NFSCHEMA_PLUGIN` subworkflow during pipeline initialization -The validation behavior is controlled through the `validation{}` scope in `nextflow.config`. +In Parts 3 and 4 we set `validate_params = false` so the pipeline could run before we had configured any schemas. +Now that we're ready to add validation, the first step is to turn it on. -Since we'll be working on parameter validation first (this section) and won't configure the input data schema until section 2, we need to temporarily tell nf-schema to skip validating the `input` parameter's file contents. +Open `nextflow.config` and find the `validate_params` parameter (around line 37), and set it to `true`: -Open `nextflow.config` and find the `validation` block (around line 247). Add `ignoreParams` to skip input file validation: +=== "After" + + ```groovy title="nextflow.config" hl_lines="1" linenums="37" + validate_params = true + ``` + +=== "Before" + + ```groovy title="nextflow.config" hl_lines="1" linenums="37" + validate_params = false + ``` + +The validation behavior itself is controlled through the `validation{}` scope in `nextflow.config`. + +Since we'll be working on parameter validation first (this section) and won't configure the input data schema until section 2, we also need to temporarily tell nf-schema to skip validating the `input` parameter's file contents. + +Find the `validation` block (around line 252) and add `ignoreParams` to skip input file validation: === "After" - ```groovy title="nextflow.config" hl_lines="3" linenums="247" + ```groovy title="nextflow.config" hl_lines="3" linenums="252" validation { defaultIgnoreParams = ["genomes"] ignoreParams = ['input'] @@ -166,7 +183,7 @@ Open `nextflow.config` and find the `validation` block (around line 247). Add `i === "Before" - ```groovy title="nextflow.config" linenums="247" + ```groovy title="nextflow.config" linenums="252" validation { defaultIgnoreParams = ["genomes"] monochromeLogs = params.monochrome_logs @@ -179,7 +196,7 @@ This configuration tells nf-schema to: - **`ignoreParams`**: Skip validation of the `input` parameter's file contents (temporary; we'll re-enable this in section 2) - **`monochromeLogs`**: Disable colored output in validation messages when set to `true` (controlled by `params.monochrome_logs`) -!!! note "Why ignore the input parameter?" +!!! info "Why ignore the input parameter?" The `input` parameter in `nextflow_schema.json` has `"schema": "assets/schema_input.json"` which tells nf-schema to validate the *contents* of the input CSV file against that schema. Since we haven't configured that schema yet, we temporarily ignore this validation. @@ -261,7 +278,7 @@ You should see something like this: | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 3.5.2 - https://nf-co.re + nf-core/tools version 4.0.2 - https://nf-co.re INFO [✓] Default parameters match schema validation INFO [✓] Pipeline schema looks valid (found 17 params) @@ -350,7 +367,7 @@ nextflow run . --outdir test-results -profile docker ```console ERROR ~ Validation of pipeline parameters failed! - -- Check '.nextflow.log' file for details + -- Check '.nextflow.log' file for details The following invalid input values have been detected: * Missing required parameter(s): input, batch @@ -638,7 +655,7 @@ Open `nextflow.config` and remove the `ignoreParams` line from the `validation` === "After" - ```groovy title="nextflow.config" linenums="247" + ```groovy title="nextflow.config" linenums="252" validation { defaultIgnoreParams = ["genomes"] monochromeLogs = params.monochrome_logs @@ -647,7 +664,7 @@ Open `nextflow.config` and remove the `ignoreParams` line from the `validation` === "Before" - ```groovy title="nextflow.config" hl_lines="3" linenums="247" + ```groovy title="nextflow.config" hl_lines="3" linenums="252" validation { defaultIgnoreParams = ["genomes"] ignoreParams = ['input'] @@ -664,7 +681,7 @@ Let's verify that our validation works by testing both valid and invalid inputs. #### 2.7.1. Test with valid input First, confirm the pipeline runs successfully with valid input. -Note that we no longer need `--validate_params false` since validation is working! +With `validate_params = true` and the input schema in place, both parameter and input data validation now run for real. ```bash nextflow run . --outdir core-hello-results -profile test,docker diff --git a/docs/en/docs/hello_nf-core/img/nf-core-demo-subway-cropped.png b/docs/en/docs/hello_nf-core/img/nf-core-demo-subway-cropped.png index ac07a8bc49..b3cdbf4790 100644 Binary files a/docs/en/docs/hello_nf-core/img/nf-core-demo-subway-cropped.png and b/docs/en/docs/hello_nf-core/img/nf-core-demo-subway-cropped.png differ diff --git a/docs/en/docs/hello_nf-core/img/search-results.png b/docs/en/docs/hello_nf-core/img/search-results.png index d25ba7f91f..379c61c554 100644 Binary files a/docs/en/docs/hello_nf-core/img/search-results.png and b/docs/en/docs/hello_nf-core/img/search-results.png differ diff --git a/hello-nf-core/custom.config b/hello-nf-core/custom.config new file mode 100644 index 0000000000..ca08f44d8e --- /dev/null +++ b/hello-nf-core/custom.config @@ -0,0 +1,9 @@ +process { + withName: 'FASTQC' { + cpus = 2 + memory = 4.GB + } + withName: 'SEQTK_TRIM' { + ext.args = '-b 5' + } +} diff --git a/hello-nf-core/malformed_samplesheet.csv b/hello-nf-core/malformed_samplesheet.csv new file mode 100644 index 0000000000..4e743f0ae3 --- /dev/null +++ b/hello-nf-core/malformed_samplesheet.csv @@ -0,0 +1,2 @@ +sample,fastq_2 +SAMPLE1,/not/a/real/file.fastq.gz diff --git a/hello-nf-core/my_params.yml b/hello-nf-core/my_params.yml new file mode 100644 index 0000000000..0142125ab9 --- /dev/null +++ b/hello-nf-core/my_params.yml @@ -0,0 +1 @@ +skip_trim: true diff --git a/hello-nf-core/solutions/core-hello-part2/.nf-core.yml b/hello-nf-core/solutions/core-hello-part2/.nf-core.yml index 4b27e1dd63..a17e2d83a5 100644 --- a/hello-nf-core/solutions/core-hello-part2/.nf-core.yml +++ b/hello-nf-core/solutions/core-hello-part2/.nf-core.yml @@ -1,6 +1,6 @@ repository_type: pipeline -nf_core_version: 3.5.2 +nf_core_version: 4.0.2 lint: files_unchanged: diff --git a/hello-nf-core/solutions/core-hello-part2/conf/base.config b/hello-nf-core/solutions/core-hello-part2/conf/base.config index e0fe40762f..d0dfc7d1d1 100644 --- a/hello-nf-core/solutions/core-hello-part2/conf/base.config +++ b/hello-nf-core/solutions/core-hello-part2/conf/base.config @@ -15,7 +15,7 @@ process { memory = { 6.GB * task.attempt } time = { 4.h * task.attempt } - errorStrategy = { task.exitStatus in ((130..145) + 104 + 175) ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in ((130..145) + 104 + (175..177)) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' diff --git a/hello-nf-core/solutions/core-hello-part2/main.nf b/hello-nf-core/solutions/core-hello-part2/main.nf index eb8d91361f..4bf5934fc5 100644 --- a/hello-nf-core/solutions/core-hello-part2/main.nf +++ b/hello-nf-core/solutions/core-hello-part2/main.nf @@ -36,7 +36,8 @@ workflow CORE_HELLO { // WORKFLOW: Run pipeline // HELLO ( - samplesheet + samplesheet, + params.outdir, ) } /* @@ -73,7 +74,6 @@ workflow { // SUBWORKFLOW: Run completion tasks // PIPELINE_COMPLETION ( - params.outdir, params.monochrome_logs, ) } diff --git a/hello-nf-core/solutions/core-hello-part2/modules.json b/hello-nf-core/solutions/core-hello-part2/modules.json index 0ef0a5149b..e897cf840e 100644 --- a/hello-nf-core/solutions/core-hello-part2/modules.json +++ b/hello-nf-core/solutions/core-hello-part2/modules.json @@ -15,12 +15,12 @@ }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "271e7fc14eb1320364416d996fb077421f3faed2", + "git_sha": "a3fb7351b1fdb2b1de282b765816bbea190e86a8", "installed_by": ["subworkflows"] }, "utils_nfschema_plugin": { "branch": "master", - "git_sha": "4b406a74dc0449c0401ed87d5bfff4252fd277fd", + "git_sha": "fdc08b8b1ae74f56686ce21f7ea11ad11990ce57", "installed_by": ["subworkflows"] } } diff --git a/hello-nf-core/solutions/core-hello-part2/nextflow.config b/hello-nf-core/solutions/core-hello-part2/nextflow.config index c29d19a6df..c8048f3143 100644 --- a/hello-nf-core/solutions/core-hello-part2/nextflow.config +++ b/hello-nf-core/solutions/core-hello-part2/nextflow.config @@ -34,9 +34,13 @@ params { config_profile_url = null // Schema validation default options - validate_params = true + validate_params = false } +// Backwards compatibility for publishDir syntax +outputDir = params.outdir +workflow.output.mode = params.publish_dir_mode + // Load base.config by default for all pipelines includeConfig 'conf/base.config' @@ -234,14 +238,14 @@ manifest { description = """A basic nf-core style version of Hello Nextflow""" mainScript = 'main.nf' defaultBranch = 'master' - nextflowVersion = '!>=25.04.0' + nextflowVersion = '!>=25.10.4' version = '1.0.0dev' doi = '' } // Nextflow plugins plugins { - id 'nf-schema@2.5.1' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-schema@2.7.2' // Validation of pipeline parameters and creation of an input channel from a sample sheet } validation { diff --git a/hello-nf-core/solutions/core-hello-part2/subworkflows/local/utils_nfcore_hello_pipeline/main.nf b/hello-nf-core/solutions/core-hello-part2/subworkflows/local/utils_nfcore_hello_pipeline/main.nf index 1be4d08e8d..661797280d 100644 --- a/hello-nf-core/solutions/core-hello-part2/subworkflows/local/utils_nfcore_hello_pipeline/main.nf +++ b/hello-nf-core/solutions/core-hello-part2/subworkflows/local/utils_nfcore_hello_pipeline/main.nf @@ -95,11 +95,9 @@ workflow PIPELINE_INITIALISATION { workflow PIPELINE_COMPLETION { take: - outdir // path: Path to output directory where results will be published monochrome_logs // boolean: Disable ANSI colour codes in log output main: - summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") // // Completion email and summary @@ -110,7 +108,7 @@ workflow PIPELINE_COMPLETION { } workflow.onError { - log.error "Pipeline failed. Please refer to troubleshooting docs: https://nf-co.re/docs/usage/troubleshooting" + log.error "Pipeline failed. Please refer to troubleshooting docs: https://nf-co.re/docs/running/troubleshooting" } } diff --git a/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index 2f30e9a463..afca543909 100644 --- a/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -17,7 +17,7 @@ workflow UTILS_NFCORE_PIPELINE { checkProfileProvided(nextflow_cli_args) emit: - valid_config + valid_config = valid_config } /* @@ -353,67 +353,3 @@ def completionSummary(monochrome_logs=true) { log.info("-${colors.purple}[${workflow.manifest.name}]${colors.red} Pipeline completed with errors${colors.reset}-") } } - -// -// Construct and send a notification to a web server as JSON e.g. Microsoft Teams and Slack -// -def imNotification(summary_params, hook_url) { - def summary = [:] - summary_params - .keySet() - .sort() - .each { group -> - summary << summary_params[group] - } - - def misc_fields = [:] - misc_fields['start'] = workflow.start - misc_fields['complete'] = workflow.complete - misc_fields['scriptfile'] = workflow.scriptFile - misc_fields['scriptid'] = workflow.scriptId - if (workflow.repository) { - misc_fields['repository'] = workflow.repository - } - if (workflow.commitId) { - misc_fields['commitid'] = workflow.commitId - } - if (workflow.revision) { - misc_fields['revision'] = workflow.revision - } - misc_fields['nxf_version'] = workflow.nextflow.version - misc_fields['nxf_build'] = workflow.nextflow.build - misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp - - def msg_fields = [:] - msg_fields['version'] = getWorkflowVersion() - msg_fields['runName'] = workflow.runName - msg_fields['success'] = workflow.success - msg_fields['dateComplete'] = workflow.complete - msg_fields['duration'] = workflow.duration - msg_fields['exitStatus'] = workflow.exitStatus - msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - msg_fields['errorReport'] = (workflow.errorReport ?: 'None') - msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") - msg_fields['projectDir'] = workflow.projectDir - msg_fields['summary'] = summary << misc_fields - - // Render the JSON template - def engine = new groovy.text.GStringTemplateEngine() - // Different JSON depending on the service provider - // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format - def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" - def hf = new File("${workflow.projectDir}/assets/${json_path}") - def json_template = engine.createTemplate(hf).make(msg_fields) - def json_message = json_template.toString() - - // POST - def post = new URL(hook_url).openConnection() - post.setRequestMethod("POST") - post.setDoOutput(true) - post.setRequestProperty("Content-Type", "application/json") - post.getOutputStream().write(json_message.getBytes("UTF-8")) - def postRC = post.getResponseCode() - if (!postRC.equals(200)) { - log.warn(post.getErrorStream().getText()) - } -} diff --git a/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test b/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test new file mode 100644 index 0000000000..8940d32d1e --- /dev/null +++ b/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test @@ -0,0 +1,29 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NFCORE_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + workflow "UTILS_NFCORE_PIPELINE" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Should run without failures") { + + when { + workflow { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out).match() } + ) + } + } +} diff --git a/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap b/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap new file mode 100644 index 0000000000..859d1030fb --- /dev/null +++ b/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap @@ -0,0 +1,19 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + true + ], + "valid_config": [ + true + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:25.726491" + } +} \ No newline at end of file diff --git a/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfschema_plugin/main.nf index ee4738c8d1..1df8b76fba 100644 --- a/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfschema_plugin/main.nf +++ b/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -38,7 +38,7 @@ workflow UTILS_NFSCHEMA_PLUGIN { } log.info paramsHelp( help_options, - params.help instanceof String ? params.help : "", + (params.help instanceof String && params.help != "true") ? params.help : "", ) exit 0 } @@ -71,4 +71,3 @@ workflow UTILS_NFSCHEMA_PLUGIN { emit: dummy_emit = true } - diff --git a/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config b/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config index 8d8c73718a..fd71cb8f85 100644 --- a/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config +++ b/hello-nf-core/solutions/core-hello-part2/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config @@ -1,5 +1,5 @@ plugins { - id "nf-schema@2.5.1" + id "nf-schema@2.7.2" } validation { diff --git a/hello-nf-core/solutions/core-hello-part2/workflows/hello.nf b/hello-nf-core/solutions/core-hello-part2/workflows/hello.nf index 5d24486510..3bab6a6dfc 100644 --- a/hello-nf-core/solutions/core-hello-part2/workflows/hello.nf +++ b/hello-nf-core/solutions/core-hello-part2/workflows/hello.nf @@ -20,6 +20,7 @@ workflow HELLO { take: ch_samplesheet // channel: samplesheet read in from --input + outdir main: @@ -60,7 +61,7 @@ workflow HELLO { softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) .mix(topic_versions_string) .collectFile( - storeDir: "${params.outdir}/pipeline_info", + storeDir: "${outdir}/pipeline_info", name: 'hello_software_' + 'versions.yml', sort: true, newLine: true diff --git a/hello-nf-core/solutions/core-hello-part3/.nf-core.yml b/hello-nf-core/solutions/core-hello-part3/.nf-core.yml index 4b27e1dd63..a17e2d83a5 100644 --- a/hello-nf-core/solutions/core-hello-part3/.nf-core.yml +++ b/hello-nf-core/solutions/core-hello-part3/.nf-core.yml @@ -1,6 +1,6 @@ repository_type: pipeline -nf_core_version: 3.5.2 +nf_core_version: 4.0.2 lint: files_unchanged: diff --git a/hello-nf-core/solutions/core-hello-part3/conf/base.config b/hello-nf-core/solutions/core-hello-part3/conf/base.config index e0fe40762f..d0dfc7d1d1 100644 --- a/hello-nf-core/solutions/core-hello-part3/conf/base.config +++ b/hello-nf-core/solutions/core-hello-part3/conf/base.config @@ -15,7 +15,7 @@ process { memory = { 6.GB * task.attempt } time = { 4.h * task.attempt } - errorStrategy = { task.exitStatus in ((130..145) + 104 + 175) ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in ((130..145) + 104 + (175..177)) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' diff --git a/hello-nf-core/solutions/core-hello-part3/main.nf b/hello-nf-core/solutions/core-hello-part3/main.nf index eb8d91361f..4bf5934fc5 100644 --- a/hello-nf-core/solutions/core-hello-part3/main.nf +++ b/hello-nf-core/solutions/core-hello-part3/main.nf @@ -36,7 +36,8 @@ workflow CORE_HELLO { // WORKFLOW: Run pipeline // HELLO ( - samplesheet + samplesheet, + params.outdir, ) } /* @@ -73,7 +74,6 @@ workflow { // SUBWORKFLOW: Run completion tasks // PIPELINE_COMPLETION ( - params.outdir, params.monochrome_logs, ) } diff --git a/hello-nf-core/solutions/core-hello-part3/modules.json b/hello-nf-core/solutions/core-hello-part3/modules.json index 7f25f2c231..4aa9c3870c 100644 --- a/hello-nf-core/solutions/core-hello-part3/modules.json +++ b/hello-nf-core/solutions/core-hello-part3/modules.json @@ -25,14 +25,14 @@ }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "271e7fc14eb1320364416d996fb077421f3faed2", + "git_sha": "a3fb7351b1fdb2b1de282b765816bbea190e86a8", "installed_by": [ "subworkflows" ] }, "utils_nfschema_plugin": { "branch": "master", - "git_sha": "4b406a74dc0449c0401ed87d5bfff4252fd277fd", + "git_sha": "fdc08b8b1ae74f56686ce21f7ea11ad11990ce57", "installed_by": [ "subworkflows" ] diff --git a/hello-nf-core/solutions/core-hello-part3/nextflow.config b/hello-nf-core/solutions/core-hello-part3/nextflow.config index c29d19a6df..c8048f3143 100644 --- a/hello-nf-core/solutions/core-hello-part3/nextflow.config +++ b/hello-nf-core/solutions/core-hello-part3/nextflow.config @@ -34,9 +34,13 @@ params { config_profile_url = null // Schema validation default options - validate_params = true + validate_params = false } +// Backwards compatibility for publishDir syntax +outputDir = params.outdir +workflow.output.mode = params.publish_dir_mode + // Load base.config by default for all pipelines includeConfig 'conf/base.config' @@ -234,14 +238,14 @@ manifest { description = """A basic nf-core style version of Hello Nextflow""" mainScript = 'main.nf' defaultBranch = 'master' - nextflowVersion = '!>=25.04.0' + nextflowVersion = '!>=25.10.4' version = '1.0.0dev' doi = '' } // Nextflow plugins plugins { - id 'nf-schema@2.5.1' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-schema@2.7.2' // Validation of pipeline parameters and creation of an input channel from a sample sheet } validation { diff --git a/hello-nf-core/solutions/core-hello-part3/subworkflows/local/utils_nfcore_hello_pipeline/main.nf b/hello-nf-core/solutions/core-hello-part3/subworkflows/local/utils_nfcore_hello_pipeline/main.nf index 1be4d08e8d..661797280d 100644 --- a/hello-nf-core/solutions/core-hello-part3/subworkflows/local/utils_nfcore_hello_pipeline/main.nf +++ b/hello-nf-core/solutions/core-hello-part3/subworkflows/local/utils_nfcore_hello_pipeline/main.nf @@ -95,11 +95,9 @@ workflow PIPELINE_INITIALISATION { workflow PIPELINE_COMPLETION { take: - outdir // path: Path to output directory where results will be published monochrome_logs // boolean: Disable ANSI colour codes in log output main: - summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") // // Completion email and summary @@ -110,7 +108,7 @@ workflow PIPELINE_COMPLETION { } workflow.onError { - log.error "Pipeline failed. Please refer to troubleshooting docs: https://nf-co.re/docs/usage/troubleshooting" + log.error "Pipeline failed. Please refer to troubleshooting docs: https://nf-co.re/docs/running/troubleshooting" } } diff --git a/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index 2f30e9a463..afca543909 100644 --- a/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -17,7 +17,7 @@ workflow UTILS_NFCORE_PIPELINE { checkProfileProvided(nextflow_cli_args) emit: - valid_config + valid_config = valid_config } /* @@ -353,67 +353,3 @@ def completionSummary(monochrome_logs=true) { log.info("-${colors.purple}[${workflow.manifest.name}]${colors.red} Pipeline completed with errors${colors.reset}-") } } - -// -// Construct and send a notification to a web server as JSON e.g. Microsoft Teams and Slack -// -def imNotification(summary_params, hook_url) { - def summary = [:] - summary_params - .keySet() - .sort() - .each { group -> - summary << summary_params[group] - } - - def misc_fields = [:] - misc_fields['start'] = workflow.start - misc_fields['complete'] = workflow.complete - misc_fields['scriptfile'] = workflow.scriptFile - misc_fields['scriptid'] = workflow.scriptId - if (workflow.repository) { - misc_fields['repository'] = workflow.repository - } - if (workflow.commitId) { - misc_fields['commitid'] = workflow.commitId - } - if (workflow.revision) { - misc_fields['revision'] = workflow.revision - } - misc_fields['nxf_version'] = workflow.nextflow.version - misc_fields['nxf_build'] = workflow.nextflow.build - misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp - - def msg_fields = [:] - msg_fields['version'] = getWorkflowVersion() - msg_fields['runName'] = workflow.runName - msg_fields['success'] = workflow.success - msg_fields['dateComplete'] = workflow.complete - msg_fields['duration'] = workflow.duration - msg_fields['exitStatus'] = workflow.exitStatus - msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - msg_fields['errorReport'] = (workflow.errorReport ?: 'None') - msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") - msg_fields['projectDir'] = workflow.projectDir - msg_fields['summary'] = summary << misc_fields - - // Render the JSON template - def engine = new groovy.text.GStringTemplateEngine() - // Different JSON depending on the service provider - // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format - def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" - def hf = new File("${workflow.projectDir}/assets/${json_path}") - def json_template = engine.createTemplate(hf).make(msg_fields) - def json_message = json_template.toString() - - // POST - def post = new URL(hook_url).openConnection() - post.setRequestMethod("POST") - post.setDoOutput(true) - post.setRequestProperty("Content-Type", "application/json") - post.getOutputStream().write(json_message.getBytes("UTF-8")) - def postRC = post.getResponseCode() - if (!postRC.equals(200)) { - log.warn(post.getErrorStream().getText()) - } -} diff --git a/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test b/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test new file mode 100644 index 0000000000..8940d32d1e --- /dev/null +++ b/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test @@ -0,0 +1,29 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NFCORE_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + workflow "UTILS_NFCORE_PIPELINE" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Should run without failures") { + + when { + workflow { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out).match() } + ) + } + } +} diff --git a/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap b/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap new file mode 100644 index 0000000000..859d1030fb --- /dev/null +++ b/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap @@ -0,0 +1,19 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + true + ], + "valid_config": [ + true + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:25.726491" + } +} \ No newline at end of file diff --git a/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfschema_plugin/main.nf index ee4738c8d1..1df8b76fba 100644 --- a/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfschema_plugin/main.nf +++ b/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -38,7 +38,7 @@ workflow UTILS_NFSCHEMA_PLUGIN { } log.info paramsHelp( help_options, - params.help instanceof String ? params.help : "", + (params.help instanceof String && params.help != "true") ? params.help : "", ) exit 0 } @@ -71,4 +71,3 @@ workflow UTILS_NFSCHEMA_PLUGIN { emit: dummy_emit = true } - diff --git a/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config b/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config index 8d8c73718a..fd71cb8f85 100644 --- a/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config +++ b/hello-nf-core/solutions/core-hello-part3/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config @@ -1,5 +1,5 @@ plugins { - id "nf-schema@2.5.1" + id "nf-schema@2.7.2" } validation { diff --git a/hello-nf-core/solutions/core-hello-part3/workflows/hello.nf b/hello-nf-core/solutions/core-hello-part3/workflows/hello.nf index 4d7541491f..e8d4229e8c 100644 --- a/hello-nf-core/solutions/core-hello-part3/workflows/hello.nf +++ b/hello-nf-core/solutions/core-hello-part3/workflows/hello.nf @@ -20,6 +20,7 @@ workflow HELLO { take: ch_samplesheet // channel: samplesheet read in from --input + outdir main: @@ -69,7 +70,7 @@ workflow HELLO { softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) .mix(topic_versions_string) .collectFile( - storeDir: "${params.outdir}/pipeline_info", + storeDir: "${outdir}/pipeline_info", name: 'hello_software_' + 'versions.yml', sort: true, newLine: true diff --git a/hello-nf-core/solutions/core-hello-part4/.nf-core.yml b/hello-nf-core/solutions/core-hello-part4/.nf-core.yml index 4b27e1dd63..a17e2d83a5 100644 --- a/hello-nf-core/solutions/core-hello-part4/.nf-core.yml +++ b/hello-nf-core/solutions/core-hello-part4/.nf-core.yml @@ -1,6 +1,6 @@ repository_type: pipeline -nf_core_version: 3.5.2 +nf_core_version: 4.0.2 lint: files_unchanged: diff --git a/hello-nf-core/solutions/core-hello-part4/conf/base.config b/hello-nf-core/solutions/core-hello-part4/conf/base.config index e0fe40762f..d0dfc7d1d1 100644 --- a/hello-nf-core/solutions/core-hello-part4/conf/base.config +++ b/hello-nf-core/solutions/core-hello-part4/conf/base.config @@ -15,7 +15,7 @@ process { memory = { 6.GB * task.attempt } time = { 4.h * task.attempt } - errorStrategy = { task.exitStatus in ((130..145) + 104 + 175) ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in ((130..145) + 104 + (175..177)) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' diff --git a/hello-nf-core/solutions/core-hello-part4/main.nf b/hello-nf-core/solutions/core-hello-part4/main.nf index eb8d91361f..4bf5934fc5 100644 --- a/hello-nf-core/solutions/core-hello-part4/main.nf +++ b/hello-nf-core/solutions/core-hello-part4/main.nf @@ -36,7 +36,8 @@ workflow CORE_HELLO { // WORKFLOW: Run pipeline // HELLO ( - samplesheet + samplesheet, + params.outdir, ) } /* @@ -73,7 +74,6 @@ workflow { // SUBWORKFLOW: Run completion tasks // PIPELINE_COMPLETION ( - params.outdir, params.monochrome_logs, ) } diff --git a/hello-nf-core/solutions/core-hello-part4/modules.json b/hello-nf-core/solutions/core-hello-part4/modules.json index 7f25f2c231..4aa9c3870c 100644 --- a/hello-nf-core/solutions/core-hello-part4/modules.json +++ b/hello-nf-core/solutions/core-hello-part4/modules.json @@ -25,14 +25,14 @@ }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "271e7fc14eb1320364416d996fb077421f3faed2", + "git_sha": "a3fb7351b1fdb2b1de282b765816bbea190e86a8", "installed_by": [ "subworkflows" ] }, "utils_nfschema_plugin": { "branch": "master", - "git_sha": "4b406a74dc0449c0401ed87d5bfff4252fd277fd", + "git_sha": "fdc08b8b1ae74f56686ce21f7ea11ad11990ce57", "installed_by": [ "subworkflows" ] diff --git a/hello-nf-core/solutions/core-hello-part4/nextflow.config b/hello-nf-core/solutions/core-hello-part4/nextflow.config index c29d19a6df..c8048f3143 100644 --- a/hello-nf-core/solutions/core-hello-part4/nextflow.config +++ b/hello-nf-core/solutions/core-hello-part4/nextflow.config @@ -34,9 +34,13 @@ params { config_profile_url = null // Schema validation default options - validate_params = true + validate_params = false } +// Backwards compatibility for publishDir syntax +outputDir = params.outdir +workflow.output.mode = params.publish_dir_mode + // Load base.config by default for all pipelines includeConfig 'conf/base.config' @@ -234,14 +238,14 @@ manifest { description = """A basic nf-core style version of Hello Nextflow""" mainScript = 'main.nf' defaultBranch = 'master' - nextflowVersion = '!>=25.04.0' + nextflowVersion = '!>=25.10.4' version = '1.0.0dev' doi = '' } // Nextflow plugins plugins { - id 'nf-schema@2.5.1' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-schema@2.7.2' // Validation of pipeline parameters and creation of an input channel from a sample sheet } validation { diff --git a/hello-nf-core/solutions/core-hello-part4/subworkflows/local/utils_nfcore_hello_pipeline/main.nf b/hello-nf-core/solutions/core-hello-part4/subworkflows/local/utils_nfcore_hello_pipeline/main.nf index 1be4d08e8d..661797280d 100644 --- a/hello-nf-core/solutions/core-hello-part4/subworkflows/local/utils_nfcore_hello_pipeline/main.nf +++ b/hello-nf-core/solutions/core-hello-part4/subworkflows/local/utils_nfcore_hello_pipeline/main.nf @@ -95,11 +95,9 @@ workflow PIPELINE_INITIALISATION { workflow PIPELINE_COMPLETION { take: - outdir // path: Path to output directory where results will be published monochrome_logs // boolean: Disable ANSI colour codes in log output main: - summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") // // Completion email and summary @@ -110,7 +108,7 @@ workflow PIPELINE_COMPLETION { } workflow.onError { - log.error "Pipeline failed. Please refer to troubleshooting docs: https://nf-co.re/docs/usage/troubleshooting" + log.error "Pipeline failed. Please refer to troubleshooting docs: https://nf-co.re/docs/running/troubleshooting" } } diff --git a/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index 2f30e9a463..afca543909 100644 --- a/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -17,7 +17,7 @@ workflow UTILS_NFCORE_PIPELINE { checkProfileProvided(nextflow_cli_args) emit: - valid_config + valid_config = valid_config } /* @@ -353,67 +353,3 @@ def completionSummary(monochrome_logs=true) { log.info("-${colors.purple}[${workflow.manifest.name}]${colors.red} Pipeline completed with errors${colors.reset}-") } } - -// -// Construct and send a notification to a web server as JSON e.g. Microsoft Teams and Slack -// -def imNotification(summary_params, hook_url) { - def summary = [:] - summary_params - .keySet() - .sort() - .each { group -> - summary << summary_params[group] - } - - def misc_fields = [:] - misc_fields['start'] = workflow.start - misc_fields['complete'] = workflow.complete - misc_fields['scriptfile'] = workflow.scriptFile - misc_fields['scriptid'] = workflow.scriptId - if (workflow.repository) { - misc_fields['repository'] = workflow.repository - } - if (workflow.commitId) { - misc_fields['commitid'] = workflow.commitId - } - if (workflow.revision) { - misc_fields['revision'] = workflow.revision - } - misc_fields['nxf_version'] = workflow.nextflow.version - misc_fields['nxf_build'] = workflow.nextflow.build - misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp - - def msg_fields = [:] - msg_fields['version'] = getWorkflowVersion() - msg_fields['runName'] = workflow.runName - msg_fields['success'] = workflow.success - msg_fields['dateComplete'] = workflow.complete - msg_fields['duration'] = workflow.duration - msg_fields['exitStatus'] = workflow.exitStatus - msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - msg_fields['errorReport'] = (workflow.errorReport ?: 'None') - msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") - msg_fields['projectDir'] = workflow.projectDir - msg_fields['summary'] = summary << misc_fields - - // Render the JSON template - def engine = new groovy.text.GStringTemplateEngine() - // Different JSON depending on the service provider - // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format - def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" - def hf = new File("${workflow.projectDir}/assets/${json_path}") - def json_template = engine.createTemplate(hf).make(msg_fields) - def json_message = json_template.toString() - - // POST - def post = new URL(hook_url).openConnection() - post.setRequestMethod("POST") - post.setDoOutput(true) - post.setRequestProperty("Content-Type", "application/json") - post.getOutputStream().write(json_message.getBytes("UTF-8")) - def postRC = post.getResponseCode() - if (!postRC.equals(200)) { - log.warn(post.getErrorStream().getText()) - } -} diff --git a/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test b/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test new file mode 100644 index 0000000000..8940d32d1e --- /dev/null +++ b/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test @@ -0,0 +1,29 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NFCORE_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + workflow "UTILS_NFCORE_PIPELINE" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Should run without failures") { + + when { + workflow { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out).match() } + ) + } + } +} diff --git a/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap b/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap new file mode 100644 index 0000000000..859d1030fb --- /dev/null +++ b/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap @@ -0,0 +1,19 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + true + ], + "valid_config": [ + true + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:25.726491" + } +} \ No newline at end of file diff --git a/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfschema_plugin/main.nf index ee4738c8d1..1df8b76fba 100644 --- a/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfschema_plugin/main.nf +++ b/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -38,7 +38,7 @@ workflow UTILS_NFSCHEMA_PLUGIN { } log.info paramsHelp( help_options, - params.help instanceof String ? params.help : "", + (params.help instanceof String && params.help != "true") ? params.help : "", ) exit 0 } @@ -71,4 +71,3 @@ workflow UTILS_NFSCHEMA_PLUGIN { emit: dummy_emit = true } - diff --git a/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config b/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config index 8d8c73718a..fd71cb8f85 100644 --- a/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config +++ b/hello-nf-core/solutions/core-hello-part4/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config @@ -1,5 +1,5 @@ plugins { - id "nf-schema@2.5.1" + id "nf-schema@2.7.2" } validation { diff --git a/hello-nf-core/solutions/core-hello-part4/workflows/hello.nf b/hello-nf-core/solutions/core-hello-part4/workflows/hello.nf index 073d36235e..fdd8f4084c 100644 --- a/hello-nf-core/solutions/core-hello-part4/workflows/hello.nf +++ b/hello-nf-core/solutions/core-hello-part4/workflows/hello.nf @@ -20,6 +20,7 @@ workflow HELLO { take: ch_samplesheet // channel: samplesheet read in from --input + outdir main: @@ -66,7 +67,7 @@ workflow HELLO { softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) .mix(topic_versions_string) .collectFile( - storeDir: "${params.outdir}/pipeline_info", + storeDir: "${outdir}/pipeline_info", name: 'hello_software_' + 'versions.yml', sort: true, newLine: true diff --git a/hello-nf-core/solutions/core-hello-part5/.nf-core.yml b/hello-nf-core/solutions/core-hello-part5/.nf-core.yml index 4b27e1dd63..a17e2d83a5 100644 --- a/hello-nf-core/solutions/core-hello-part5/.nf-core.yml +++ b/hello-nf-core/solutions/core-hello-part5/.nf-core.yml @@ -1,6 +1,6 @@ repository_type: pipeline -nf_core_version: 3.5.2 +nf_core_version: 4.0.2 lint: files_unchanged: diff --git a/hello-nf-core/solutions/core-hello-part5/conf/base.config b/hello-nf-core/solutions/core-hello-part5/conf/base.config index e0fe40762f..d0dfc7d1d1 100644 --- a/hello-nf-core/solutions/core-hello-part5/conf/base.config +++ b/hello-nf-core/solutions/core-hello-part5/conf/base.config @@ -15,7 +15,7 @@ process { memory = { 6.GB * task.attempt } time = { 4.h * task.attempt } - errorStrategy = { task.exitStatus in ((130..145) + 104 + 175) ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in ((130..145) + 104 + (175..177)) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' diff --git a/hello-nf-core/solutions/core-hello-part5/main.nf b/hello-nf-core/solutions/core-hello-part5/main.nf index eb8d91361f..4bf5934fc5 100644 --- a/hello-nf-core/solutions/core-hello-part5/main.nf +++ b/hello-nf-core/solutions/core-hello-part5/main.nf @@ -36,7 +36,8 @@ workflow CORE_HELLO { // WORKFLOW: Run pipeline // HELLO ( - samplesheet + samplesheet, + params.outdir, ) } /* @@ -73,7 +74,6 @@ workflow { // SUBWORKFLOW: Run completion tasks // PIPELINE_COMPLETION ( - params.outdir, params.monochrome_logs, ) } diff --git a/hello-nf-core/solutions/core-hello-part5/modules.json b/hello-nf-core/solutions/core-hello-part5/modules.json index 6b6a752e3c..34389d9fe0 100644 --- a/hello-nf-core/solutions/core-hello-part5/modules.json +++ b/hello-nf-core/solutions/core-hello-part5/modules.json @@ -21,12 +21,12 @@ }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "271e7fc14eb1320364416d996fb077421f3faed2", + "git_sha": "a3fb7351b1fdb2b1de282b765816bbea190e86a8", "installed_by": ["subworkflows"] }, "utils_nfschema_plugin": { "branch": "master", - "git_sha": "4b406a74dc0449c0401ed87d5bfff4252fd277fd", + "git_sha": "fdc08b8b1ae74f56686ce21f7ea11ad11990ce57", "installed_by": ["subworkflows"] } } diff --git a/hello-nf-core/solutions/core-hello-part5/nextflow.config b/hello-nf-core/solutions/core-hello-part5/nextflow.config index c29d19a6df..88e6f1bc97 100644 --- a/hello-nf-core/solutions/core-hello-part5/nextflow.config +++ b/hello-nf-core/solutions/core-hello-part5/nextflow.config @@ -37,6 +37,10 @@ params { validate_params = true } +// Backwards compatibility for publishDir syntax +outputDir = params.outdir +workflow.output.mode = params.publish_dir_mode + // Load base.config by default for all pipelines includeConfig 'conf/base.config' @@ -234,14 +238,14 @@ manifest { description = """A basic nf-core style version of Hello Nextflow""" mainScript = 'main.nf' defaultBranch = 'master' - nextflowVersion = '!>=25.04.0' + nextflowVersion = '!>=25.10.4' version = '1.0.0dev' doi = '' } // Nextflow plugins plugins { - id 'nf-schema@2.5.1' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-schema@2.7.2' // Validation of pipeline parameters and creation of an input channel from a sample sheet } validation { diff --git a/hello-nf-core/solutions/core-hello-part5/subworkflows/local/utils_nfcore_hello_pipeline/main.nf b/hello-nf-core/solutions/core-hello-part5/subworkflows/local/utils_nfcore_hello_pipeline/main.nf index 07015a6615..a2252e16d4 100644 --- a/hello-nf-core/solutions/core-hello-part5/subworkflows/local/utils_nfcore_hello_pipeline/main.nf +++ b/hello-nf-core/solutions/core-hello-part5/subworkflows/local/utils_nfcore_hello_pipeline/main.nf @@ -94,11 +94,9 @@ workflow PIPELINE_INITIALISATION { workflow PIPELINE_COMPLETION { take: - outdir // path: Path to output directory where results will be published monochrome_logs // boolean: Disable ANSI colour codes in log output main: - summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") // // Completion email and summary @@ -109,7 +107,7 @@ workflow PIPELINE_COMPLETION { } workflow.onError { - log.error "Pipeline failed. Please refer to troubleshooting docs: https://nf-co.re/docs/usage/troubleshooting" + log.error "Pipeline failed. Please refer to troubleshooting docs: https://nf-co.re/docs/running/troubleshooting" } } diff --git a/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index 2f30e9a463..afca543909 100644 --- a/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -17,7 +17,7 @@ workflow UTILS_NFCORE_PIPELINE { checkProfileProvided(nextflow_cli_args) emit: - valid_config + valid_config = valid_config } /* @@ -353,67 +353,3 @@ def completionSummary(monochrome_logs=true) { log.info("-${colors.purple}[${workflow.manifest.name}]${colors.red} Pipeline completed with errors${colors.reset}-") } } - -// -// Construct and send a notification to a web server as JSON e.g. Microsoft Teams and Slack -// -def imNotification(summary_params, hook_url) { - def summary = [:] - summary_params - .keySet() - .sort() - .each { group -> - summary << summary_params[group] - } - - def misc_fields = [:] - misc_fields['start'] = workflow.start - misc_fields['complete'] = workflow.complete - misc_fields['scriptfile'] = workflow.scriptFile - misc_fields['scriptid'] = workflow.scriptId - if (workflow.repository) { - misc_fields['repository'] = workflow.repository - } - if (workflow.commitId) { - misc_fields['commitid'] = workflow.commitId - } - if (workflow.revision) { - misc_fields['revision'] = workflow.revision - } - misc_fields['nxf_version'] = workflow.nextflow.version - misc_fields['nxf_build'] = workflow.nextflow.build - misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp - - def msg_fields = [:] - msg_fields['version'] = getWorkflowVersion() - msg_fields['runName'] = workflow.runName - msg_fields['success'] = workflow.success - msg_fields['dateComplete'] = workflow.complete - msg_fields['duration'] = workflow.duration - msg_fields['exitStatus'] = workflow.exitStatus - msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - msg_fields['errorReport'] = (workflow.errorReport ?: 'None') - msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") - msg_fields['projectDir'] = workflow.projectDir - msg_fields['summary'] = summary << misc_fields - - // Render the JSON template - def engine = new groovy.text.GStringTemplateEngine() - // Different JSON depending on the service provider - // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format - def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" - def hf = new File("${workflow.projectDir}/assets/${json_path}") - def json_template = engine.createTemplate(hf).make(msg_fields) - def json_message = json_template.toString() - - // POST - def post = new URL(hook_url).openConnection() - post.setRequestMethod("POST") - post.setDoOutput(true) - post.setRequestProperty("Content-Type", "application/json") - post.getOutputStream().write(json_message.getBytes("UTF-8")) - def postRC = post.getResponseCode() - if (!postRC.equals(200)) { - log.warn(post.getErrorStream().getText()) - } -} diff --git a/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test b/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test new file mode 100644 index 0000000000..8940d32d1e --- /dev/null +++ b/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test @@ -0,0 +1,29 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NFCORE_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + workflow "UTILS_NFCORE_PIPELINE" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Should run without failures") { + + when { + workflow { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out).match() } + ) + } + } +} diff --git a/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap b/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap new file mode 100644 index 0000000000..859d1030fb --- /dev/null +++ b/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap @@ -0,0 +1,19 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + true + ], + "valid_config": [ + true + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:25.726491" + } +} \ No newline at end of file diff --git a/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfschema_plugin/main.nf index ee4738c8d1..1df8b76fba 100644 --- a/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfschema_plugin/main.nf +++ b/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -38,7 +38,7 @@ workflow UTILS_NFSCHEMA_PLUGIN { } log.info paramsHelp( help_options, - params.help instanceof String ? params.help : "", + (params.help instanceof String && params.help != "true") ? params.help : "", ) exit 0 } @@ -71,4 +71,3 @@ workflow UTILS_NFSCHEMA_PLUGIN { emit: dummy_emit = true } - diff --git a/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config b/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config index 8d8c73718a..fd71cb8f85 100644 --- a/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config +++ b/hello-nf-core/solutions/core-hello-part5/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config @@ -1,5 +1,5 @@ plugins { - id "nf-schema@2.5.1" + id "nf-schema@2.7.2" } validation { diff --git a/hello-nf-core/solutions/core-hello-part5/workflows/hello.nf b/hello-nf-core/solutions/core-hello-part5/workflows/hello.nf index 073d36235e..fdd8f4084c 100644 --- a/hello-nf-core/solutions/core-hello-part5/workflows/hello.nf +++ b/hello-nf-core/solutions/core-hello-part5/workflows/hello.nf @@ -20,6 +20,7 @@ workflow HELLO { take: ch_samplesheet // channel: samplesheet read in from --input + outdir main: @@ -66,7 +67,7 @@ workflow HELLO { softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) .mix(topic_versions_string) .collectFile( - storeDir: "${params.outdir}/pipeline_info", + storeDir: "${outdir}/pipeline_info", name: 'hello_software_' + 'versions.yml', sort: true, newLine: true diff --git a/hello-nf-core/solutions/core-hello-start/conf/base.config b/hello-nf-core/solutions/core-hello-start/conf/base.config index 1abcd9876f..d0dfc7d1d1 100644 --- a/hello-nf-core/solutions/core-hello-start/conf/base.config +++ b/hello-nf-core/solutions/core-hello-start/conf/base.config @@ -15,7 +15,7 @@ process { memory = { 6.GB * task.attempt } time = { 4.h * task.attempt } - errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in ((130..145) + 104 + (175..177)) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' @@ -59,4 +59,8 @@ process { errorStrategy = 'retry' maxRetries = 2 } + withLabel: process_gpu { + ext.use_gpu = { workflow.profile.contains('gpu') } + accelerator = { workflow.profile.contains('gpu') ? 1 : null } + } } diff --git a/hello-nf-core/solutions/core-hello-start/main.nf b/hello-nf-core/solutions/core-hello-start/main.nf index f72a236660..4bf5934fc5 100644 --- a/hello-nf-core/solutions/core-hello-start/main.nf +++ b/hello-nf-core/solutions/core-hello-start/main.nf @@ -36,7 +36,8 @@ workflow CORE_HELLO { // WORKFLOW: Run pipeline // HELLO ( - samplesheet + samplesheet, + params.outdir, ) } /* @@ -57,7 +58,10 @@ workflow { params.monochrome_logs, args, params.outdir, - params.input + params.input, + params.help, + params.help_full, + params.show_hidden ) // @@ -70,7 +74,6 @@ workflow { // SUBWORKFLOW: Run completion tasks // PIPELINE_COMPLETION ( - params.outdir, params.monochrome_logs, ) } diff --git a/hello-nf-core/solutions/core-hello-start/modules.json b/hello-nf-core/solutions/core-hello-start/modules.json index e36947ce00..2c85357275 100644 --- a/hello-nf-core/solutions/core-hello-start/modules.json +++ b/hello-nf-core/solutions/core-hello-start/modules.json @@ -4,23 +4,24 @@ "repos": { "https://github.com/nf-core/modules.git": { "modules": { - "nf-core": {} + "nf-core": { + } }, "subworkflows": { "nf-core": { "utils_nextflow_pipeline": { "branch": "master", - "git_sha": "c2b22d85f30a706a3073387f30380704fcae013b", + "git_sha": "05954dab2ff481bcb999f24455da29a5828af08d", "installed_by": ["subworkflows"] }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "51ae5406a030d4da1e49e4dab49756844fdd6c7a", + "git_sha": "a3fb7351b1fdb2b1de282b765816bbea190e86a8", "installed_by": ["subworkflows"] }, "utils_nfschema_plugin": { "branch": "master", - "git_sha": "2fd2cd6d0e7b273747f32e465fdc6bcc3ae0814e", + "git_sha": "fdc08b8b1ae74f56686ce21f7ea11ad11990ce57", "installed_by": ["subworkflows"] } } diff --git a/hello-nf-core/solutions/core-hello-start/nextflow.config b/hello-nf-core/solutions/core-hello-start/nextflow.config index d633adb989..08e1e1670f 100644 --- a/hello-nf-core/solutions/core-hello-start/nextflow.config +++ b/hello-nf-core/solutions/core-hello-start/nextflow.config @@ -22,7 +22,9 @@ params { show_hidden = false version = false pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' - trace_report_suffix = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')// Config options + trace_report_suffix = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') + + // Config options config_profile_name = null config_profile_description = null @@ -35,6 +37,10 @@ params { validate_params = true } +// Backwards compatibility for publishDir syntax +outputDir = params.outdir +workflow.output.mode = params.publish_dir_mode + // Load base.config by default for all pipelines includeConfig 'conf/base.config' @@ -75,7 +81,18 @@ profiles { apptainer.enabled = false docker.runOptions = '-u $(id -u):$(id -g)' } - arm { + arm64 { + process.arch = 'arm64' + // TODO https://github.com/nf-core/modules/issues/6694 + // For now if you're using arm64 you have to use wave for the sake of the maintainers + // wave profile + apptainer.ociAutoPull = true + singularity.ociAutoPull = true + wave.enabled = true + wave.freeze = true + wave.strategy = 'conda,container' + } + emulate_amd64 { docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } singularity { @@ -132,16 +149,25 @@ profiles { wave.freeze = true wave.strategy = 'conda,container' } + gpu { + docker.runOptions = '-u $(id -u):$(id -g) --gpus all' + apptainer.runOptions = '--nv' + singularity.runOptions = '--nv' + } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } } -// Load nf-core custom profiles from different Institutions -includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/nfcore_custom.config" : "/dev/null" +// Load nf-core custom profiles from different institutions + +// If params.custom_config_base is set AND either the NXF_OFFLINE environment variable is not set or params.custom_config_base is a local path, the nfcore_custom.config file from the specified base path is included. +// Load core/hello custom profiles from different institutions. +includeConfig params.custom_config_base && (!System.getenv('NXF_OFFLINE') || !params.custom_config_base.startsWith('http')) ? "${params.custom_config_base}/nfcore_custom.config" : "/dev/null" + // Load core/hello custom profiles from different institutions. // TODO nf-core: Optionally, you can add a pipeline-specific nf-core config at https://github.com/nf-core/configs -// includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/pipeline/hello.config" : "/dev/null" +// includeConfig params.custom_config_base && (!System.getenv('NXF_OFFLINE') || !params.custom_config_base.startsWith('http')) ? "${params.custom_config_base}/pipeline/hello.config" : "/dev/null" // Set default registry for Apptainer, Docker, Podman, Charliecloud and Singularity independent of -profile // Will not be used unless Apptainer / Docker / Podman / Charliecloud / Singularity are enabled @@ -197,11 +223,10 @@ dag { manifest { name = 'core/hello' - author = """GG""" // The author field is deprecated from Nextflow version 24.10.0, use contributors instead contributors = [ // TODO nf-core: Update the field with the details of the contributors to your pipeline. New with Nextflow version 24.10.0 [ - name: 'GG', + name: 'your-name', affiliation: '', email: '', github: '', @@ -210,29 +235,22 @@ manifest { ], ] homePage = 'https://github.com/core/hello' - description = """basic nf-core style version of Hello Nextflow""" + description = """A basic nf-core style version of Hello Nextflow""" mainScript = 'main.nf' - defaultBranch = 'main' - nextflowVersion = '!>=24.04.2' + defaultBranch = 'master' + nextflowVersion = '!>=25.10.4' version = '1.0.0dev' doi = '' } // Nextflow plugins plugins { - id 'nf-schema@2.2.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-schema@2.7.2' // Validation of pipeline parameters and creation of an input channel from a sample sheet } validation { defaultIgnoreParams = ["genomes"] monochromeLogs = params.monochrome_logs - help { - enabled = true - command = "nextflow run core/hello -profile --input samplesheet.csv --outdir " - fullParameter = "help_full" - showHiddenParameter = "show_hidden" - } } - // Load modules.config for DSL2 module specific options includeConfig 'conf/modules.config' diff --git a/hello-nf-core/solutions/core-hello-start/nextflow_schema.json b/hello-nf-core/solutions/core-hello-start/nextflow_schema.json index 5ee5ec357f..dcc4cdf462 100644 --- a/hello-nf-core/solutions/core-hello-start/nextflow_schema.json +++ b/hello-nf-core/solutions/core-hello-start/nextflow_schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/core/hello/main/nextflow_schema.json", + "$id": "https://raw.githubusercontent.com/core/hello/master/nextflow_schema.json", "title": "core/hello pipeline parameters", - "description": "basic nf-core style version of Hello Nextflow", + "description": "A basic nf-core style version of Hello Nextflow", "type": "object", "$defs": { "input_output_options": { @@ -98,14 +98,7 @@ "description": "Method used to save pipeline results to output directory.", "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", "fa_icon": "fas fa-copy", - "enum": [ - "symlink", - "rellink", - "link", - "copy", - "copyNoFollow", - "move" - ], + "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], "hidden": true }, "monochrome_logs": { @@ -133,6 +126,18 @@ "fa_icon": "far calendar", "description": "Suffix to add to the trace report filename. Default is the date and time in the format yyyy-MM-dd_HH-mm-ss.", "hidden": true + }, + "help": { + "type": ["boolean", "string"], + "description": "Display the help message." + }, + "help_full": { + "type": "boolean", + "description": "Display the full detailed help message." + }, + "show_hidden": { + "type": "boolean", + "description": "Display hidden parameters in the help message (only works when --help or --help_full are provided)." } } } diff --git a/hello-nf-core/solutions/core-hello-start/subworkflows/local/utils_nfcore_hello_pipeline/main.nf b/hello-nf-core/solutions/core-hello-start/subworkflows/local/utils_nfcore_hello_pipeline/main.nf index b4aafb69bb..c65866b8a8 100644 --- a/hello-nf-core/solutions/core-hello-start/subworkflows/local/utils_nfcore_hello_pipeline/main.nf +++ b/hello-nf-core/solutions/core-hello-start/subworkflows/local/utils_nfcore_hello_pipeline/main.nf @@ -11,6 +11,7 @@ include { UTILS_NFSCHEMA_PLUGIN } from '../../nf-core/utils_nfschema_plugin' include { paramsSummaryMap } from 'plugin/nf-schema' include { samplesheetToList } from 'plugin/nf-schema' +include { paramsHelp } from 'plugin/nf-schema' include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' @@ -30,6 +31,9 @@ workflow PIPELINE_INITIALISATION { nextflow_cli_args // array: List of positional nextflow CLI args outdir // string: The output directory where the results will be saved input // string: Path to input samplesheet + help // boolean: Display help message and exit + help_full // boolean: Show the full help message + show_hidden // boolean: Show hidden parameters in the help message main: @@ -48,10 +52,25 @@ workflow PIPELINE_INITIALISATION { // // Validate parameters and generate parameter summary to stdout // + + def before_text = "" + def after_text = "" + if (monochrome_logs) { + before_text = before_text.replaceAll(/\033\[[0-9;]*m/, '') + } + + command = "nextflow run ${workflow.manifest.name} -profile --input samplesheet.csv --outdir " + UTILS_NFSCHEMA_PLUGIN ( workflow, validate_params, - null + null, + help, + help_full, + show_hidden, + before_text, + after_text, + command ) // @@ -65,8 +84,8 @@ workflow PIPELINE_INITIALISATION { // Create channel from input file provided through params.input // - Channel - .fromList(samplesheetToList(params.input, "${projectDir}/assets/schema_input.json")) + channel + .fromList(samplesheetToList(input, "${projectDir}/assets/schema_input.json")) .map { meta, fastq_1, fastq_2 -> if (!fastq_2) { @@ -99,11 +118,9 @@ workflow PIPELINE_INITIALISATION { workflow PIPELINE_COMPLETION { take: - outdir // path: Path to output directory where results will be published monochrome_logs // boolean: Disable ANSI colour codes in log output main: - summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") // // Completion email and summary @@ -111,10 +128,11 @@ workflow PIPELINE_COMPLETION { workflow.onComplete { completionSummary(monochrome_logs) + } workflow.onError { - log.error "Pipeline failed. Please refer to troubleshooting docs: https://nf-co.re/docs/usage/troubleshooting" + log.error "Pipeline failed. Please refer to troubleshooting docs for common issues: https://nf-co.re/docs/running/troubleshooting" } } diff --git a/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index 2f30e9a463..afca543909 100644 --- a/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -17,7 +17,7 @@ workflow UTILS_NFCORE_PIPELINE { checkProfileProvided(nextflow_cli_args) emit: - valid_config + valid_config = valid_config } /* @@ -353,67 +353,3 @@ def completionSummary(monochrome_logs=true) { log.info("-${colors.purple}[${workflow.manifest.name}]${colors.red} Pipeline completed with errors${colors.reset}-") } } - -// -// Construct and send a notification to a web server as JSON e.g. Microsoft Teams and Slack -// -def imNotification(summary_params, hook_url) { - def summary = [:] - summary_params - .keySet() - .sort() - .each { group -> - summary << summary_params[group] - } - - def misc_fields = [:] - misc_fields['start'] = workflow.start - misc_fields['complete'] = workflow.complete - misc_fields['scriptfile'] = workflow.scriptFile - misc_fields['scriptid'] = workflow.scriptId - if (workflow.repository) { - misc_fields['repository'] = workflow.repository - } - if (workflow.commitId) { - misc_fields['commitid'] = workflow.commitId - } - if (workflow.revision) { - misc_fields['revision'] = workflow.revision - } - misc_fields['nxf_version'] = workflow.nextflow.version - misc_fields['nxf_build'] = workflow.nextflow.build - misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp - - def msg_fields = [:] - msg_fields['version'] = getWorkflowVersion() - msg_fields['runName'] = workflow.runName - msg_fields['success'] = workflow.success - msg_fields['dateComplete'] = workflow.complete - msg_fields['duration'] = workflow.duration - msg_fields['exitStatus'] = workflow.exitStatus - msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - msg_fields['errorReport'] = (workflow.errorReport ?: 'None') - msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") - msg_fields['projectDir'] = workflow.projectDir - msg_fields['summary'] = summary << misc_fields - - // Render the JSON template - def engine = new groovy.text.GStringTemplateEngine() - // Different JSON depending on the service provider - // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format - def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" - def hf = new File("${workflow.projectDir}/assets/${json_path}") - def json_template = engine.createTemplate(hf).make(msg_fields) - def json_message = json_template.toString() - - // POST - def post = new URL(hook_url).openConnection() - post.setRequestMethod("POST") - post.setDoOutput(true) - post.setRequestProperty("Content-Type", "application/json") - post.getOutputStream().write(json_message.getBytes("UTF-8")) - def postRC = post.getResponseCode() - if (!postRC.equals(200)) { - log.warn(post.getErrorStream().getText()) - } -} diff --git a/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test b/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test new file mode 100644 index 0000000000..8940d32d1e --- /dev/null +++ b/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test @@ -0,0 +1,29 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NFCORE_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + workflow "UTILS_NFCORE_PIPELINE" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Should run without failures") { + + when { + workflow { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out).match() } + ) + } + } +} diff --git a/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap b/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap new file mode 100644 index 0000000000..859d1030fb --- /dev/null +++ b/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap @@ -0,0 +1,19 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + true + ], + "valid_config": [ + true + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:25.726491" + } +} \ No newline at end of file diff --git a/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfschema_plugin/main.nf index 93de2a5245..1df8b76fba 100644 --- a/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfschema_plugin/main.nf +++ b/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -4,6 +4,7 @@ include { paramsSummaryLog } from 'plugin/nf-schema' include { validateParameters } from 'plugin/nf-schema' +include { paramsHelp } from 'plugin/nf-schema' workflow UTILS_NFSCHEMA_PLUGIN { @@ -15,29 +16,56 @@ workflow UTILS_NFSCHEMA_PLUGIN { // when this input is empty it will automatically use the configured schema or // "${projectDir}/nextflow_schema.json" as default. This input should not be empty // for meta pipelines + help // boolean: show help message + help_full // boolean: show full help message + show_hidden // boolean: show hidden parameters in help message + before_text // string: text to show before the help message and parameters summary + after_text // string: text to show after the help message and parameters summary + command // string: an example command of the pipeline main: + if(help || help_full) { + help_options = [ + beforeText: before_text, + afterText: after_text, + command: command, + showHidden: show_hidden, + fullHelp: help_full, + ] + if(parameters_schema) { + help_options << [parametersSchema: parameters_schema] + } + log.info paramsHelp( + help_options, + (params.help instanceof String && params.help != "true") ? params.help : "", + ) + exit 0 + } + // // Print parameter summary to stdout. This will display the parameters // that differ from the default given in the JSON schema // + + summary_options = [:] if(parameters_schema) { - log.info paramsSummaryLog(input_workflow, parameters_schema:parameters_schema) - } else { - log.info paramsSummaryLog(input_workflow) + summary_options << [parametersSchema: parameters_schema] } + log.info before_text + log.info paramsSummaryLog(summary_options, input_workflow) + log.info after_text // // Validate the parameters using nextflow_schema.json or the schema // given via the validation.parametersSchema configuration option // if(validate_params) { + validateOptions = [:] if(parameters_schema) { - validateParameters(parameters_schema:parameters_schema) - } else { - validateParameters() + validateOptions << [parametersSchema: parameters_schema] } + validateParameters(validateOptions) } emit: diff --git a/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config b/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config index 478fb8a05f..fd71cb8f85 100644 --- a/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config +++ b/hello-nf-core/solutions/core-hello-start/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config @@ -1,5 +1,5 @@ plugins { - id "nf-schema@2.1.0" + id "nf-schema@2.7.2" } validation { diff --git a/hello-nf-core/solutions/core-hello-start/workflows/hello.nf b/hello-nf-core/solutions/core-hello-start/workflows/hello.nf index d54c260170..e788a1414f 100644 --- a/hello-nf-core/solutions/core-hello-start/workflows/hello.nf +++ b/hello-nf-core/solutions/core-hello-start/workflows/hello.nf @@ -16,14 +16,16 @@ workflow HELLO { take: ch_samplesheet // channel: samplesheet read in from --input + outdir + main: - ch_versions = channel.empty() + def ch_versions = channel.empty() // // Collate and save software versions // - def topic_versions = Channel.topic("versions") + def topic_versions = channel.topic("versions") .distinct() .branch { entry -> versions_file: entry instanceof Path @@ -40,19 +42,16 @@ workflow HELLO { "${process}:\n${tool_versions.join('\n')}" } - softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) + def ch_collated_versions = softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) .mix(topic_versions_string) .collectFile( - storeDir: "${params.outdir}/pipeline_info", + storeDir: "${outdir}/pipeline_info", name: 'hello_software_' + 'versions.yml', sort: true, newLine: true - ).set { ch_collated_versions } - - + ) emit: versions = ch_versions // channel: [ path(versions.yml) ] - } /*