Workflow params (part 1)

Signed-off-by: Ben Sherman <[email protected]>

Author: bentsherman

docs/cli.md

@@ -297,8 +297,8 @@ Parameters are applied in the following order (from lowest to highest priority):
 
 1. Parameters defined in pipeline scripts (e.g. `main.nf`)
 2. Parameters defined in {ref}`config files <config-params>`
-6. Parameters specified in a params file (`-params-file`)
-7. Parameters specified on the command line (`--something value`)
+3. Parameters specified in a params file (`-params-file`)
+4. Parameters specified on the command line (`--something value`)
 
 ## Managing projects
 

docs/reference/syntax.md

@@ -28,7 +28,8 @@ A Nextflow script may contain the following top-level declarations:
 - Shebang
 - Feature flags
 - Include declarations
-- Parameter declarations
+- Params block
+- Parameter declarations (legacy)
 - Workflow definitions
 - Process definitions
 - Function definitions
@@ -107,9 +108,22 @@ The following definitions can be included:
 - Processes
 - Named workflows
 
-### Parameter
+### Params block
 
-A parameter declaration is an assignment. The target should be a pipeline parameter and the source should be an expression:
+The params block consists of one or more *parameter declarations*. A parameter declaration consists of a name and an optional default value:
+
+```nextflow
+params {
+    input
+    save_intermeds = false
+}
+```
+
+Only one params block may be defined in a script.
+
+### Parameter (legacy)
+
+A legacy parameter declaration is an assignment. The target should be a pipeline parameter and the source should be an expression:
 
 ```nextflow
 params.message = 'Hello world!'

docs/vscode.md

@@ -26,6 +26,19 @@ The language server parses scripts and config files according to the {ref}`Nextf
 
 When you hover over certain source code elements, such as variable names and function calls, the extension provides a tooltip with related information, such as the definition and/or documentation for the element.
 
+If a [Javadoc](https://en.wikipedia.org/wiki/Javadoc) comment is defined above a workflow, process, function, type definition, or parameter in a `params` block, the extension will include the contents of the comment in hover hints. The following is an example Javadoc comment:
+
+```nextflow
+/**
+ * Say hello to someone.
+ *
+ * @param name
+ */
+def sayHello(name) {
+    println "Hello, ${name}!"
+}
+```
+
 ### Code navigation
 
 The **Outline** section in the Explorer panel lists top-level definitions when you view a script. Include declarations in scripts and config files act as links, and ctrl-clicking them opens the corresponding script or config file.

docs/workflow.md

@@ -22,18 +22,67 @@ workflow {
 }
 ```
 
-### Parameters
+## Parameters
 
-Parameters can be defined in the script with a default value that can be overridden from the CLI, params file, or config file. Parameters should only be used by the entry workflow:
+Parameters can be declared in a Nextflow script with the `params` block or with *legacy* parameter declarations.
+
+### Params block
+
+:::{versionadded} 25.05.0-edge
+:::
+
+:::{note}
+This feature requires the {ref}`strict syntax <strict-syntax-page>` to be enabled (`NXF_SYNTAX_PARSER=v2`).
+:::
+
+A script can declare parameters using the `params` block:
+
+```nextflow
+params {
+    /**
+     * Path to input data.
+     */
+    input
+
+    /**
+     * Whether to save intermediate files.
+     */
+    save_intermeds = false
+}
+```
+
+Parameters can be used in the entry workflow:
+
+```nextflow
+workflow {
+    if( params.input )
+        analyze(params.input, params.save_intermeds)
+    else
+        analyze(fake_input(), params.save_intermeds)
+}
+```
+
+:::{note}
+While params can be used outside the entry workflow, Nextflow will not be able to validate them at compile-time. Only params used in the entry workflow are validated against the params definition. Params can be passed to workflows and processes as explicit inputs to enable compile-time validation.
+:::
+
+The default value can be overridden by the command line, params file, or config file. Parameters from multiple sources are resolved in the order described in {ref}`cli-params`.
+
+A parameter that doesn't specify a default value is a *required* param. If a required param is not given a value at runtime, the run will fail.
+
+### Legacy parameters
+
+Parameters can be declared by assigning a `params` property to a default value:
 
 ```nextflow
-params.data = '/some/data/file'
+params.input = '/some/data/file'
+params.save_intermeds = false
 
 workflow {
-    if( params.data )
-        bar(params.data)
+    if( params.input )
+        analyze(params.input, params.save_intermeds)
     else
-        bar(foo())
+        analyze(fake_input(), params.save_intermeds)
 }
 ```
 

modules/nextflow/src/main/groovy/nextflow/Session.groovy

@@ -113,6 +113,16 @@ class Session implements ISession {
      */
     ScriptBinding binding
 
+    /**
+     * Params that were specified on the command line.
+     */
+    Map cliParams
+
+    /**
+     * Params that were specified in the configuration.
+     */
+    Map configParams
+
     /**
      * Holds the configuration object
      */
@@ -418,7 +428,7 @@ class Session implements ISession {
     /**
      * Initialize the session workDir, libDir, baseDir and scriptName variables
      */
-    Session init( ScriptFile scriptFile, List<String> args=null ) {
+    Session init( ScriptFile scriptFile, List<String> args=null, Map<String,?> cliParams=null, Map<String,?> configParams=null ) {
 
         if(!workDir.mkdirs()) throw new AbortOperationException("Cannot create work-dir: $workDir -- Make sure you have write permissions or specify a different directory by using the `-w` command line option")
         log.debug "Work-dir: ${workDir.toUriString()} [${FileHelper.getPathFsType(workDir)}]"
@@ -445,6 +455,8 @@ class Session implements ISession {
         this.workflowMetadata = new WorkflowMetadata(this, scriptFile)
 
         // configure script params
+        this.cliParams = cliParams
+        this.configParams = configParams
         binding.setParams( (Map)config.params )
         binding.setArgs( new ScriptRunner.ArgsList(args) )
 

modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy

@@ -325,12 +325,18 @@ class CmdRun extends CmdBase implements HubOptions {
         // -- specify the arguments
         final scriptFile = getScriptFile(pipeline)
 
+        // -- load command line params
+        final baseDir = scriptFile.parent
+        final cliParams = parsedParams(ConfigBuilder.getConfigVars(baseDir))
+
         // create the config object
         final builder = new ConfigBuilder()
                 .setOptions(launcher.options)
                 .setCmdRun(this)
-                .setBaseDir(scriptFile.parent)
-        final config = builder .build()
+                .setBaseDir(baseDir)
+                .setCliParams(cliParams)
+        final config = builder.build()
+        final configParams = builder.getConfigParams()
 
         // check DSL syntax in the config
         launchInfo(config, scriptFile)
@@ -376,7 +382,7 @@ class CmdRun extends CmdBase implements HubOptions {
         }
 
         // -- run it!
-        runner.execute(scriptArgs, this.entryName)
+        runner.execute(scriptArgs, cliParams, configParams, this.entryName)
     }
 
     protected void printBanner() {

modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy

@@ -61,6 +61,8 @@ class ConfigBuilder {
 
     Path currentDir
 
+    Map<String,?> cliParams
+
     boolean showAllProfiles
 
     String profile = DEFAULT_PROFILE
@@ -81,9 +83,11 @@ class ConfigBuilder {
 
     Map<String,String> env = new HashMap<>(System.getenv())
 
-    List<String> warnings = new ArrayList<>(10);
+    List<String> warnings = new ArrayList<>(10)
+
+    Map<String,Object> declaredParams = [:]
 
-    {
+    ConfigBuilder() {
         setHomeDir(Const.APP_HOME_DIR)
         setCurrentDir(Paths.get('.'))
     }
@@ -114,6 +118,11 @@ class ConfigBuilder {
         return this
     }
 
+    ConfigBuilder setCliParams( Map<String,?> cliParams ) {
+        this.cliParams = cliParams
+        return this
+    }
+
     ConfigBuilder setBaseDir( Path path ) {
         this.baseDir = path.complete()
         return this
@@ -162,6 +171,10 @@ class ConfigBuilder {
         return this
     }
 
+    Map<String,Object> getConfigParams() {
+        return declaredParams
+    }
+
     static private wrapValue( value ) {
         if( !value )
             return ''
@@ -327,11 +340,11 @@ class ConfigBuilder {
         // this is needed to make sure to reuse the same
         // instance of the config vars across different instances of the ConfigBuilder
         // and prevent multiple parsing of the same params file (which can even be remote resource)
-        return cacheableConfigVars(baseDir)
+        return getConfigVars(baseDir)
     }
 
     @Memoized
-    static private Map cacheableConfigVars(Path base) {
+    static Map getConfigVars(Path base) {
         final binding = new HashMap(10)
         binding.put('baseDir', base)
         binding.put('projectDir', base)
@@ -351,8 +364,8 @@ class ConfigBuilder {
                 .setIgnoreIncludes(ignoreIncludes)
         ConfigObject result = new ConfigObject()
 
-        if( cmdRun && (cmdRun.hasParams()) )
-            parser.setParams(cmdRun.parsedParams(configVars()))
+        if( cliParams )
+            parser.setParams(cliParams)
 
         // add the user specified environment to the session env
         env.sort().each { name, value -> result.env.put(name,value) }
@@ -383,7 +396,7 @@ class ConfigBuilder {
             }
 
             if( validateProfile ) {
-                checkValidProfile(parser.getProfiles())
+                checkValidProfile(parser.getDeclaredProfiles())
             }
 
         }
@@ -414,9 +427,10 @@ class ConfigBuilder {
             parser.setProfiles(profile.tokenize(','))
         }
 
-        final config = parse0(parser, entry)
+        def config = parse0(parser, entry)
         if( NF.getSyntaxParserVersion() == 'v1' )
             validate(config, entry)
+        declaredParams.putAll(parser.getDeclaredParams())
         result.merge(config)
     }
 
@@ -734,8 +748,8 @@ class ConfigBuilder {
         }
 
         // -- add the command line parameters to the 'taskConfig' object
-        if( cmdRun.hasParams() )
-            config.params = mergeMaps( (Map)config.params, cmdRun.parsedParams(configVars()), NF.strictMode )
+        if( cliParams )
+            config.params = mergeMaps( (Map)config.params, cliParams, NF.strictMode )
 
         if( cmdRun.withoutDocker && config.docker instanceof Map ) {
             // disable docker execution

modules/nextflow/src/main/groovy/nextflow/config/ConfigParser.groovy

@@ -67,6 +67,11 @@ interface ConfigParser {
      */
     ConfigParser setParams(Map vars)
 
+    /**
+     * Set the profiles that should be applied.
+     */
+    ConfigParser setProfiles(List<String> profiles)
+
     /**
      * Parse a config object from the given source.
      */
@@ -75,13 +80,13 @@ interface ConfigParser {
     ConfigObject parse(Path path)
 
     /**
-     * Set the profiles that should be applied.
+     * Get the set of declared profiles.
      */
-    ConfigParser setProfiles(List<String> profiles)
+    Set<String> getDeclaredProfiles()
 
     /**
-     * Get the set of available profiles.
+     * Get the map of declared params.
      */
-    Set<String> getProfiles()
+    Map<String,Object> getDeclaredParams()
 
 }

modules/nextflow/src/main/groovy/nextflow/config/parser/v1/ConfigParserV1.groovy

@@ -123,10 +123,15 @@ class ConfigParserV1 implements ConfigParser {
     }
 
     @Override
-    Set<String> getProfiles() {
+    Set<String> getDeclaredProfiles() {
         Collections.unmodifiableSet(conditionalNames)
     }
 
+    @Override
+    Map<String,Object> getDeclaredParams() {
+        [:]
+    }
+
     private Grengine getGrengine() {
         if( grengine ) {
             return grengine