POC: Global Cache

docs/reference/env-vars.md

@@ -118,6 +118,12 @@ The following environment variables control the configuration of the Nextflow ru
 : The file storage path against which relative file paths are resolved.
 : For example, with `NXF_FILE_ROOT=/some/root/path`, the use of `file('hello')` will be resolved to the absolute path `/some/root/path/hello`. A remote root path can be specified using the usual protocol prefix, e.g. `NXF_FILE_ROOT=s3://my-bucket/data`. Files defined using an absolute path are not affected by this setting.
 
+`NXF_GLOBALCACHE_PATH`
+: :::{versionadded} 26.04.0
+  :::
+: Enable global caching, using the given path as cache root.
+: This allows cached tasks to be re-used by different runs. Enabling global caching implicitly enables resume, and sets the work directory to `${NXF_GLOBALCACHE_PATH}/work`.
+
 `NXF_HOME`
 : Nextflow home directory (default: `$HOME/.nextflow`).
 

modules/nextflow/src/main/groovy/nextflow/Session.groovy

@@ -426,7 +426,7 @@ class Session implements ISession {
             return null
         final String path = cloudcache.path
         final result = path ? FileHelper.asPath(path) : workDir
-        if( result.scheme !in ['s3','az','gs'] ) {
+        if( result.scheme !in ['file','s3','az','gs'] ) {
             throw new IllegalArgumentException("Storage path not supported by Cloud-cache - offending value: '${result}'")
         }
         return result

modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy

@@ -33,6 +33,7 @@ import nextflow.cli.CmdNode
 import nextflow.cli.CmdRun
 import nextflow.exception.AbortOperationException
 import nextflow.exception.ConfigParseException
+import nextflow.file.FileHelper
 import nextflow.secret.SecretsLoader
 import nextflow.secret.SecretsProvider
 import nextflow.util.HistoryFile
@@ -754,6 +755,24 @@ class ConfigBuilder {
         if( cliParams )
             config.params = mergeMaps( (Map)config.params, cliParams, NF.strictMode )
 
+        // -- use global cache if specified
+        final globalCachePath = env.get('NXF_GLOBALCACHE_PATH')
+        if( globalCachePath ) {
+            final sessionId = new UUID(0L, 0L).toString()
+            config.resume = sessionId
+            config.cloudcache = [
+                enabled: true,
+                path: "${globalCachePath}/cache"
+            ]
+            config.workDir = "${globalCachePath}/work"
+
+            log.warn "Enabling global cache. This will enable resume, set the cache path to ${config.cloudcache.path}, and set the work directory to ${config.workDir}"
+
+            final cachePath = FileHelper.asPath(globalCachePath)
+            cachePath.resolve("cache/${sessionId}").mkdirs()
+        }
+
+        // -- set container options
         if( cmdRun.withoutDocker && config.docker instanceof Map ) {
             // disable docker execution
             log.debug "Disabling execution in Docker container as requested by command-line option `-without-docker`"

modules/nextflow/src/main/groovy/nextflow/processor/TaskHasher.groovy

@@ -50,11 +50,11 @@ class TaskHasher {
 
         final keys = new ArrayList<Object>()
 
-        // add session UUID
-        keys << session.uniqueId
-
-        // add fully-qualified process name
-        keys << task.processor.name
+        // add session id and process name if global caching is not enabled
+        if( session.uniqueId != new UUID(0L, 0L) ) {
+            keys << session.uniqueId
+            keys << task.processor.name
+        }
 
         // add source code of `script:` or `exec:` block
         //

modules/nextflow/src/main/groovy/nextflow/script/ScriptRunner.groovy

@@ -260,7 +260,7 @@ class ScriptRunner {
         }
 
         // -- when resume, make sure the session id exists in the executions history
-        if( session.resumeMode && !HistoryFile.DEFAULT.checkExistsById(session.uniqueId.toString()) ) {
+        if( session.resumeMode && session.uniqueId != new UUID(0L, 0L) && !HistoryFile.DEFAULT.checkExistsById(session.uniqueId.toString()) ) {
             throw new AbortOperationException("Can't find a run with the specified id: ${session.uniqueId} -- Execution can't be resumed")
         }