Workflow outputs in Side Quests

docs/en/docs/side_quests/debugging.md

@@ -2439,7 +2439,6 @@ Now it's time to put the systematic debugging approach into practice. The workfl
         * Process with input/output mismatch
         */
         process processFiles {
-            publishDir "${params.output}/processed", mode: 'copy'
 
             input:
                 tuple val(sample_id), path(input_file)
@@ -2458,7 +2457,6 @@ Now it's time to put the systematic debugging approach into practice. The workfl
         * Process with resource issues
         */
         process heavyProcess {
-            publishDir "${params.output}/heavy", mode: 'copy'
 
             time '100 s'
 
@@ -2481,7 +2479,6 @@ Now it's time to put the systematic debugging approach into practice. The workfl
         * Process with file handling issues
         */
         process handleFiles {
-            publishDir "${params.output}/files", mode: 'copy'
 
             input:
                 path input_file
@@ -2501,7 +2498,7 @@ Now it's time to put the systematic debugging approach into practice. The workfl
         * Main workflow with channel issues
         */
         workflow {
-
+            main:
             // Channel with incorrect usage
             input_ch = channel
                 .fromPath(params.input)
@@ -2513,6 +2510,26 @@ Now it's time to put the systematic debugging approach into practice. The workfl
             heavy_ch = heavyProcess(input_ch.map{it[0]})
 
             handleFiles(heavyProcess.out)
+
+            publish:
+            processed = processFiles.out
+            heavy = heavyProcess.out
+            files = handleFiles.out
+        }
+
+        output {
+            directory params.output
+            mode 'copy'
+
+            processed {
+                path 'processed'
+            }
+            heavy {
+                path 'heavy'
+            }
+            files {
+                path 'files'
+            }
         }
         ```
 

docs/en/docs/side_quests/dev_environment.md

@@ -283,7 +283,6 @@ Efficient navigation is crucial when working with complex workflows spanning mul
     ```groovy title="basic_workflow.nf" linenums="3"
     process FASTQC {
         tag "${sample_id}"
-        publishDir "${params.output_dir}/fastqc", mode: 'copy'
 
         input:
         tuple val(sample_id), path(reads)
@@ -417,13 +416,13 @@ This is invaluable when:
 
 Sometimes you need to find where specific patterns are used across your entire project. Press `Ctrl/Cmd+Shift+F` to open the search panel.
 
-Try searching for `publishDir` across the workspace:
+Try searching for `container` across the workspace:
 
 ![Project search](img/project_search.png)
 
-This shows you every file that uses publish directories, helping you:
+This shows you every file that uses the container directive, helping you:
 
-- Understand output organization patterns
+- Understand which processes use containers
 - Find examples of specific directives
 - Ensure consistency across modules
 

docs/en/docs/side_quests/essential_scripting_patterns.md

@@ -1001,8 +1001,6 @@ Take a look a the module file `modules/generate_report.nf`:
 ```groovy title="modules/generate_report.nf" linenums="1"
 process GENERATE_REPORT {
 
-    publishDir 'results/reports', mode: 'copy'
-
     input:
     tuple val(meta), path(reads)
 

docs/en/docs/side_quests/metadata.md

@@ -887,8 +887,6 @@ You can open the module file to examine its code:
 // Generate ASCII art with cowpy
 process COWPY {
 
-    publishDir "results/", mode: 'copy'
-
     container 'community.wave.seqera.io/library/cowpy:1.1.5--3db457ae1977a273'
 
     input:

docs/en/docs/side_quests/nf-test.md

@@ -119,8 +119,6 @@ You can see the full workflow code below.
     */
     process sayHello {
 
-        publishDir 'results', mode: 'copy'
-
         input:
             val greeting
 
@@ -138,8 +136,6 @@ You can see the full workflow code below.
     */
     process convertToUpper {
 
-        publishDir 'results', mode: 'copy'
-
         input:
             path input_file
 
@@ -153,7 +149,7 @@ You can see the full workflow code below.
     }
 
     workflow {
-
+        main:
         // create a channel for inputs from a CSV file
         greeting_ch = channel.fromPath(params.input_file).splitCsv().flatten()
 
@@ -162,6 +158,15 @@ You can see the full workflow code below.
 
         // convert the greeting to uppercase
         convertToUpper(sayHello.out)
+
+        publish:
+        greetings = sayHello.out
+        upper_greetings = convertToUpper.out
+    }
+
+    output {
+        directory 'results'
+        mode 'copy'
     }
     ```
 

docs/en/docs/side_quests/working_with_files.md

@@ -1568,18 +1568,14 @@ Make the following edit to the workflow:
 You can open the module file to examine its code:
 
 ```groovy title="modules/analyze_reads.nf - process example" linenums="1"
-#!/usr/bin/env nextflow
-
 process ANALYZE_READS {
     tag { meta.id }
 
-    publishDir { "results/${meta.id}" }, mode: 'copy'
-
     input:
     tuple val(meta), path(files)
 
     output:
-    tuple val(meta.id), path("${meta.id}_stats.txt")
+    tuple val(meta), path("${meta.id}_stats.txt")
 
     script:
     """
@@ -1596,9 +1592,8 @@ process ANALYZE_READS {
 
 !!! note
 
-    The `tag` and `publishDir` directives use closure syntax (`{ ... }`) instead of string interpolation (`"${...}"`).
-    This is because these directives reference input variables (`meta`) that aren't available until runtime.
-    The closure syntax defers evaluation until the process actually runs.
+    The `tag` directive uses closure syntax (`{ ... }`) because it references input variables (`meta`) that aren't available until the process runs.
+    The closure defers evaluation until runtime.
 
 !!! note
 
@@ -1689,20 +1684,75 @@ Now let's actually call the `ANALYZE_READS` process on the `ch_samples` channel.
 
 In the main workflow, make the following code changes:
 
+1. Add a `main:` label at the top of the workflow block (before the existing channel code)
+2. Replace the temporary `.view()` call with the process call
+3. Add a `publish:` section to the workflow
+4. Add an `output {}` block after the workflow
+
 === "After"
 
-    ```groovy title="main.nf" linenums="23"
+    ```groovy title="main.nf" linenums="5" hl_lines="2 20 23 24 27 28 29 30 31"
+    workflow {
+        main:
+        // Load files with channel.fromFilePairs
+        ch_files = channel.fromFilePairs('data/patientA_rep1_normal_R{1,2}_001.fastq.gz')
+        ch_files.map { id,  files ->
+           def (sample, replicate, type, readNum) = id.tokenize('_')
+           [
+               [
+                   id: sample,
+                   replicate: replicate.replace('rep', ''),
+                   type: type
+               ],
+               files
+           ]
+        }
+            .set { ch_samples }
+
         // Run the analysis
         ANALYZE_READS(ch_samples)
+
+        publish:
+        analysis_results = ANALYZE_READS.out
+    }
+
+    output {
+        directory 'results'
+        mode 'copy'
+
+        analysis_results {
+            path { meta, file -> "${meta.id}" }
+        }
+    }
     ```
 
 === "Before"
 
-    ```groovy title="main.nf" linenums="23"
+    ```groovy title="main.nf" linenums="5"
+    workflow {
+        // Load files with channel.fromFilePairs
+        ch_files = channel.fromFilePairs('data/patientA_rep1_normal_R{1,2}_001.fastq.gz')
+        ch_files.map { id,  files ->
+           def (sample, replicate, type, readNum) = id.tokenize('_')
+           [
+               [
+                   id: sample,
+                   replicate: replicate.replace('rep', ''),
+                   type: type
+               ],
+               files
+           ]
+        }
+            .set { ch_samples }
+
         // Temporary: peek into ch_samples
         ch_samples.view()
+    }
     ```
 
+The `publish:` section declares which process outputs to publish, and the `output {}` block configures where and how they are published.
+The `path` closure in the `output {}` block receives each output element and determines the subdirectory structure. Here we use `meta.id` to organize results by patient.
+
 Let's run this:
 
 ```bash
@@ -1720,7 +1770,7 @@ nextflow run main.nf
     [b5/110360] process > ANALYZE_READS (patientA) [100%] 1 of 1 ✔
     ```
 
-This process is set up to publish its outputs to a `results` directory, so have a look in there.
+The outputs are published to a `results` directory, so have a look in there.
 
 ??? abstract "Directory and file contents"
 
@@ -1809,18 +1859,22 @@ We are overwriting the output file each time.
 
 Since we have access to the patient metadata, we can use it to make the published files unique by including differentiating metadata, either in the directory structure or in the filenames themselves.
 
-Make the following change to the workflow:
+Make the following change to the `output {}` block:
 
 === "After"
 
-    ```groovy title="modules/analyze_reads.nf" linenums="6"
-        publishDir { "results/${meta.type}/${meta.id}/${meta.replicate}" }, mode: 'copy'
+    ```groovy title="main.nf" hl_lines="3"
+    analysis_results {
+        path { meta, file -> "${meta.type}/${meta.id}/${meta.replicate}" }
+    }
     ```
 
 === "Before"
 
-    ```groovy title="modules/analyze_reads.nf" linenums="6"
-        publishDir { "results/${meta.id}" }, mode: 'copy'
+    ```groovy title="main.nf" hl_lines="3"
+    analysis_results {
+        path { meta, file -> "${meta.id}" }
+    }
     ```
 
 Here we show the option of using additional directory levels to account for sample types and replicates, but you could experiment with doing it at the filename level as well.
@@ -1888,7 +1942,7 @@ You can learn more about this in the [Metadata and meta maps](./metadata.md) sid
 
 ### Takeaway
 
-- The `publishDir` directive can organize outputs based on metadata values
+- The `output {}` block can organize outputs based on metadata values using dynamic path closures
 - Metadata in tuples enables structured organization of results
 - This approach creates maintainable workflows with clear data provenance
 - Processes can take tuples of metadata and files as input
@@ -1996,7 +2050,7 @@ Applying these techniques in your own work will enable you to build more efficie
     ch_pairs = channel.fromFilePairs('data/*_R{1,2}_001.fastq.gz')
     ```
 
-6.  **Using File Operations in Processes:** We integrated file operations into Nextflow processes with proper input handling, using `publishDir` to organize outputs based on metadata.
+6.  **Using File Operations in Processes:** We integrated file operations into Nextflow processes with proper input handling, using the `output {}` block to organize outputs based on metadata.
 
     - Associate a meta map with the process inputs
 
@@ -2021,7 +2075,14 @@ Applying these techniques in your own work will enable you to build more efficie
     - Organize outputs based on metadata
 
     ```groovy
-    publishDir { "results/${meta.type}/${meta.id}/${meta.replicate}" }, mode: 'copy'
+    output {
+        directory 'results'
+        mode 'copy'
+
+        analysis_results {
+            path { meta, file -> "${meta.type}/${meta.id}/${meta.replicate}" }
+        }
+    }
     ```
 
 ### Additional resources

side-quests/debugging/buggy_workflow.nf

@@ -15,7 +15,6 @@ params{
  * Process with input/output mismatch
  */
 process processFiles {
-    publishDir "${params.output}/processed", mode: 'copy'
 
     input:
     tuple val(sample_id), path(input_file)
@@ -33,7 +32,6 @@ process processFiles {
  * Process with resource issues
  */
 process heavyProcess {
-    publishDir "${params.output}/heavy", mode: 'copy'
 
     time '1 ms'
 
@@ -56,7 +54,6 @@ process heavyProcess {
  * Process with file handling issues
  */
 process handleFiles {
-    publishDir "${params.output}/files", mode: 'copy'
 
     input:
     path input_file
@@ -89,4 +86,24 @@ workflow {
 
     file_ch = channel.fromPath("*.txt")
     handleFiles(file_ch)
+
+    publish:
+    processed = processFiles.out
+    heavy = heavyProcess.out
+    files = handleFiles.out
+}
+
+output {
+    directory params.output
+    mode 'copy'
+
+    processed {
+        path 'processed'
+    }
+    heavy {
+        path 'heavy'
+    }
+    files {
+        path 'files'
+    }
 }

side-quests/essential_scripting_patterns/modules/generate_report.nf

@@ -1,7 +1,5 @@
 process GENERATE_REPORT {
 
-    publishDir 'results/reports', mode: 'copy'
-
     input:
     tuple val(meta), path(reads)