diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdLineageTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/CmdLineageTest.groovy index 93afb60821..50e0814355 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/CmdLineageTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/CmdLineageTest.groovy @@ -227,20 +227,19 @@ class CmdLineageTest extends Specification { 'this is a script', null,null, null, null, null, [:],[], null) lidFile5.text = encoder.encode(entry) - final network = """flowchart BT - lid://12345/file.bam@{shape: document, label: "lid://12345/file.bam"} - lid://123987/file.bam@{shape: document, label: "lid://123987/file.bam"} - lid://123987@{shape: process, label: "foo [lid://123987]"} - ggal_gut@{shape: document, label: "ggal_gut"} - lid://45678/output.txt@{shape: document, label: "lid://45678/output.txt"} - lid://45678@{shape: process, label: "bar [lid://45678]"} - - lid://123987/file.bam -->lid://12345/file.bam - lid://123987 -->lid://123987/file.bam - ggal_gut -->lid://123987 - lid://45678/output.txt -->lid://123987 - lid://45678 -->lid://45678/output.txt -""" + final network = """\ + flowchart TB + lid://12345/file.bam["lid://12345/file.bam"] + lid://123987/file.bam["lid://123987/file.bam"] + lid://123987(["foo [lid://123987]"]) + ggal_gut["ggal_gut"] + lid://45678/output.txt["lid://45678/output.txt"] + lid://45678(["bar [lid://45678]"]) + lid://123987/file.bam --> lid://12345/file.bam + lid://123987 --> lid://123987/file.bam + ggal_gut --> lid://123987 + lid://45678/output.txt --> lid://123987 + lid://45678 --> lid://45678/output.txt""".stripIndent() final template = MermaidHtmlRenderer.readTemplate() def expectedOutput = template.replace('REPLACE_WITH_NETWORK_DATA', network) @@ -256,7 +255,7 @@ class CmdLineageTest extends Specification { then: stdout.size() == 1 - stdout[0] == "Linage graph for lid://12345/file.bam rendered in ${outputHtml}" + stdout[0] == "Rendered lineage graph for lid://12345/file.bam to ${outputHtml}" outputHtml.exists() outputHtml.text == expectedOutput diff --git a/modules/nf-lineage/src/main/nextflow/lineage/cli/LinCommandImpl.groovy b/modules/nf-lineage/src/main/nextflow/lineage/cli/LinCommandImpl.groovy index da61c2cd40..c5e05f5070 100644 --- a/modules/nf-lineage/src/main/nextflow/lineage/cli/LinCommandImpl.groovy +++ b/modules/nf-lineage/src/main/nextflow/lineage/cli/LinCommandImpl.groovy @@ -21,21 +21,16 @@ import static nextflow.lineage.fs.LinPath.* import java.nio.charset.StandardCharsets import java.nio.file.Path -import groovy.transform.Canonical import groovy.transform.CompileStatic import nextflow.Session import nextflow.cli.CmdLineage import nextflow.config.ConfigMap -import nextflow.dag.MermaidHtmlRenderer import nextflow.lineage.LinHistoryRecord import nextflow.lineage.LinPropertyValidator import nextflow.lineage.LinStore import nextflow.lineage.LinStoreFactory import nextflow.lineage.LinUtils -import nextflow.lineage.model.FileOutput import nextflow.lineage.model.Parameter -import nextflow.lineage.model.TaskRun -import nextflow.lineage.model.WorkflowRun import nextflow.lineage.serde.LinEncoder import nextflow.ui.TableBuilder import org.eclipse.jgit.diff.DiffAlgorithm @@ -52,14 +47,7 @@ class LinCommandImpl implements CmdLineage.LinCommand { private static final Path DEFAULT_HTML_FILE = Path.of("lineage.html") - @Canonical - static class Edge { - String source - String destination - String label - } - - static final private String ERR_NOT_LOADED = 'Error lineage store not loaded - Check Nextflow configuration' + private static final String ERR_NOT_LOADED = 'Error lineage store not loaded - Check Nextflow configuration' @Override void list(ConfigMap config) { @@ -119,139 +107,13 @@ class LinCommandImpl implements CmdLineage.LinCommand { } try { final renderFile = args.size() > 1 ? Path.of(args[1]) : DEFAULT_HTML_FILE - renderLineage(store, args[0], renderFile) - println("Linage graph for ${args[0]} rendered in $renderFile") + new LinDagRenderer(store).render(args[0], renderFile) + println("Rendered lineage graph for ${args[0]} to $renderFile") } catch (Throwable e) { println("ERROR: rendering lineage graph - ${e.message}") } } - private void renderLineage(LinStore store, String dataLid, Path file) { - def lines = [] as List - lines << "flowchart BT".toString() - final nodesToRender = new LinkedList() - nodesToRender.add(dataLid) - final edgesToRender = new LinkedList() - while (!nodesToRender.isEmpty()) { - final node = nodesToRender.removeFirst() - processNode(lines, node, nodesToRender, edgesToRender, store) - } - lines << "" - edgesToRender.each { lines << " ${it.source} -->${it.destination}".toString() } - lines << "" - lines.join('\n') - final template = MermaidHtmlRenderer.readTemplate() - file.text = template.replace('REPLACE_WITH_NETWORK_DATA', lines.join('\n')) - } - - private String safeId( String rawId){ - return rawId.replaceAll(/[^a-zA-Z0-9_.:\/\-]/, '_') - } - - private void processNode(List lines, String nodeToRender, LinkedList nodes, LinkedList edges, LinStore store) { - if (!isLidUri(nodeToRender)) - throw new Exception("Identifier is not a lineage URL") - final key = nodeToRender.substring(LID_PROT.size()) - final lidObject = store.load(key) - switch (lidObject.getClass()) { - case FileOutput: - processDataOutput(lidObject as FileOutput, lines, nodeToRender, nodes, edges) - break; - - case WorkflowRun: - processWorkflowRun(lidObject as WorkflowRun, lines, nodeToRender, edges) - break - - case TaskRun: - processTaskRun(lidObject as TaskRun, lines, nodeToRender, nodes, edges) - break - - default: - throw new Exception("Unrecognized type reference ${lidObject.getClass().getSimpleName()}") - } - } - - private void processTaskRun(TaskRun taskRun, List lines, String nodeToRender, LinkedList nodes, LinkedList edges) { - lines << " ${nodeToRender}@{shape: process, label: \"${taskRun.name} [$nodeToRender]\"}".toString() - final parameters = taskRun.input - for (Parameter source : parameters) { - if (source.type.equals("path")) { - manageFileInParam(lines, nodeToRender, nodes, edges, source.value) - } else { - final label = convertToLabel(source.value.toString()) - final id = safeId(source.value.toString()) - lines << " ${id}@{shape: document, label: \"${label}\"}".toString(); - edges.add(new Edge(id, nodeToRender)) - } - } - } - - private void processWorkflowRun(WorkflowRun wfRun, List lines, String nodeToRender, LinkedList edges) { - lines << """ ${nodeToRender}@{shape: processes, label: \"${wfRun.name} [${nodeToRender}]\"}""".toString() - final parameters = wfRun.params - parameters.each { - final label = convertToLabel(it.value.toString()) - final id = safeId(it.value.toString()) - lines << " ${id}@{shape: document, label: \"${label}\"}".toString(); - edges.add(new Edge(id, nodeToRender)) - } - } - - private void processDataOutput(FileOutput lidObject, List lines, String nodeToRender, LinkedList nodes, LinkedList edges){ - lines << " ${nodeToRender}@{shape: document, label: \"${nodeToRender}\"}".toString(); - final source = lidObject.source - if(! source ) - return - if (isLidUri(source)) { - nodes.add(source) - edges.add(new Edge(source, nodeToRender)) - } else { - final label = convertToLabel(source) - final id = safeId(source) - lines << " ${id}@{shape: document, label: \"${label}\"}".toString(); - edges.add(new Edge(id, nodeToRender)) - } - } - - private String convertToLabel(String label){ - return label.replace('http', 'h\u200Ettp') - } - - private void manageFileInParam(List lines, String nodeToRender, LinkedList nodes, LinkedList edges, value){ - if (value instanceof Collection) { - value.each { manageFileInParam(lines, nodeToRender, nodes, edges, it) } - return - } - if (value instanceof CharSequence) { - final source = value.toString() - if (isLidUri(source)) { - nodes.add(source) - edges.add(new Edge(source, nodeToRender)) - return - } - } - if (value instanceof Map ) { - if (value.path) { - final path = value.path.toString() - if (isLidUri(path)) { - nodes.add(path) - edges.add(new Edge(path, nodeToRender)) - return - } else { - final label = convertToLabel(path) - final id = safeId(path) - lines << " ${id}@{shape: document, label: \"${label}\"}".toString(); - edges.add(new Edge(id, nodeToRender)) - return - } - } - } - final label = convertToLabel(value.toString()) - final id = safeId(value.toString()) - lines << " ${id}@{shape: document, label: \"${label}\"}".toString(); - edges.add(new Edge(id, nodeToRender)) - } - @Override void diff(ConfigMap config, List args) { if (!isLidUri(args[0]) || !isLidUri(args[1])) diff --git a/modules/nf-lineage/src/main/nextflow/lineage/cli/LinDagRenderer.groovy b/modules/nf-lineage/src/main/nextflow/lineage/cli/LinDagRenderer.groovy new file mode 100644 index 0000000000..60546e8b50 --- /dev/null +++ b/modules/nf-lineage/src/main/nextflow/lineage/cli/LinDagRenderer.groovy @@ -0,0 +1,203 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.lineage.cli + +import java.nio.file.Path + +import groovy.transform.Canonical +import groovy.transform.CompileStatic +import nextflow.dag.MermaidHtmlRenderer +import nextflow.lineage.LinStore +import nextflow.lineage.model.FileOutput +import nextflow.lineage.model.TaskRun +import nextflow.lineage.model.WorkflowRun + +import static nextflow.lineage.fs.LinPath.LID_PROT +import static nextflow.lineage.fs.LinPath.isLidUri +/** + * Renderer for the lineage graph. + * + * @author Ben Sherman + */ +@CompileStatic +class LinDagRenderer { + + private final LinStore store + + private List queue + + private List nodes + + private List edges + + LinDagRenderer(LinStore store) { + this.store = store + } + + private void enqueue(String lid) { + queue.add(lid) + } + + private void addNode(String id, String label, NodeType type) { + nodes.add(new Node(id, label, type)) + } + + private void addEdge(String source, String target) { + edges.add(new Edge(source, target)) + } + + void render(String lid, Path file) { + // visit nodes + this.queue = new LinkedList() + this.nodes = new LinkedList() + this.edges = new LinkedList() + + enqueue(lid) + while( !queue.isEmpty() ) + visitLid(queue.pop()) + + // render Mermaid diagram + final lines = new LinkedList() + lines << "flowchart TB" + + for( final node : nodes ) { + if( node.type == NodeType.FILE ) + lines << " ${node.id}[\"${node.label}\"]".toString() + if( node.type == NodeType.TASK ) + lines << " ${node.id}([\"${node.label}\"])".toString() + } + + for( final edge : edges ) + lines << " ${edge.source} --> ${edge.target}".toString() + + final template = MermaidHtmlRenderer.readTemplate() + file.text = template.replace('REPLACE_WITH_NETWORK_DATA', lines.join('\n')) + } + + private void visitLid(String lid) { + if( !isLidUri(lid) ) + throw new Exception("Identifier is not a lineage URL: ${lid}") + final record = store.load(rawLid(lid)) + if( record instanceof FileOutput ) + visitFileOutput(lid, record) + else if( record instanceof TaskRun ) + visitTaskRun(lid, record) + else if( record instanceof WorkflowRun ) + visitWorkflowRun(lid, record) + else + throw new Exception("Cannot render lineage for type ${record.getClass().getSimpleName()} -- must be a FileOutput, TaskRun, or WorkflowRun") + } + + private void visitFileOutput(String lid, FileOutput fileOutput) { + addNode(lid, lid, NodeType.FILE) + final source = fileOutput.source + if( !source ) + return + if( isLidUri(source) ) { + enqueue(source) + addEdge(source, lid) + } + else { + final id = safeId(source) + final label = safeLabel(source) + addNode(id, label, NodeType.FILE) + addEdge(id, lid) + } + } + + private void visitTaskRun(String lid, TaskRun taskRun) { + addNode(lid, "${taskRun.name} [${lid}]", NodeType.TASK) + for( final param : taskRun.input ) { + visitParameter(lid, param.value) + } + } + + private void visitWorkflowRun(String lid, WorkflowRun workflowRun) { + addNode(lid, "${workflowRun.name} [${lid}]", NodeType.TASK) + for( final param : workflowRun.params ) { + visitParameter0(lid, param.value.toString()) + } + } + + private void visitParameter(String lid, Object value) { + if( value instanceof Collection ) { + for( final el : value ) + visitParameter(lid, el) + } + else if( value instanceof CharSequence ) { + final source = value.toString() + if( isLidUri(source) ) { + enqueue(source) + addEdge(source, lid) + } + else { + visitParameter0(lid, source) + } + } + else if( value instanceof Map && value.path ) { + final path = value.path.toString() + if( isLidUri(path) ) { + enqueue(path) + addEdge(path, lid) + } + else { + visitParameter0(lid, path) + } + } + else { + visitParameter0(lid, value.toString()) + } + } + + private void visitParameter0(String lid, String value) { + final id = safeId(value) + final label = safeLabel(value) + addNode(id, label, NodeType.FILE) + addEdge(id, lid) + } + + private static String rawLid(String lid) { + return lid.substring(LID_PROT.size()) + } + + private static String safeId(String rawId) { + return rawId.replaceAll(/[^a-zA-Z0-9_.:\/\-]/, '_') + } + + private static String safeLabel(String label) { + return label.replace('http', 'h\u200Ettp') + } + + @Canonical + private static class Node { + String id + String label + NodeType type + } + + @Canonical + private static class Edge { + String source + String target + } + + private static enum NodeType { + FILE, + TASK, + } + +} diff --git a/modules/nf-lineage/src/test/nextflow/lineage/cli/LinCommandImplTest.groovy b/modules/nf-lineage/src/test/nextflow/lineage/cli/LinCommandImplTest.groovy index 078042852f..0d7c8767af 100644 --- a/modules/nf-lineage/src/test/nextflow/lineage/cli/LinCommandImplTest.groovy +++ b/modules/nf-lineage/src/test/nextflow/lineage/cli/LinCommandImplTest.groovy @@ -246,22 +246,21 @@ class LinCommandImplTest extends Specification{ 'this is a script', null,null, null, null, null, [:],[]) lidFile5.text = encoder.encode(entry) - final network = """flowchart BT - lid://12345/file.bam@{shape: document, label: "lid://12345/file.bam"} - lid://123987/file.bam@{shape: document, label: "lid://123987/file.bam"} - lid://123987@{shape: process, label: "foo [lid://123987]"} - ggal_gut@{shape: document, label: "ggal_gut"} - path/to/file@{shape: document, label: "path/to/file"} - lid://45678/output.txt@{shape: document, label: "lid://45678/output.txt"} - lid://45678@{shape: process, label: "bar [lid://45678]"} - - lid://123987/file.bam -->lid://12345/file.bam - lid://123987 -->lid://123987/file.bam - ggal_gut -->lid://123987 - lid://45678/output.txt -->lid://123987 - path/to/file -->lid://123987 - lid://45678 -->lid://45678/output.txt -""" + final network = """\ + flowchart TB + lid://12345/file.bam["lid://12345/file.bam"] + lid://123987/file.bam["lid://123987/file.bam"] + lid://123987(["foo [lid://123987]"]) + ggal_gut["ggal_gut"] + path/to/file["path/to/file"] + lid://45678/output.txt["lid://45678/output.txt"] + lid://45678(["bar [lid://45678]"]) + lid://123987/file.bam --> lid://12345/file.bam + lid://123987 --> lid://123987/file.bam + ggal_gut --> lid://123987 + lid://45678/output.txt --> lid://123987 + path/to/file --> lid://123987 + lid://45678 --> lid://45678/output.txt""".stripIndent() final template = MermaidHtmlRenderer.readTemplate() def expectedOutput = template.replace('REPLACE_WITH_NETWORK_DATA', network) @@ -276,7 +275,7 @@ class LinCommandImplTest extends Specification{ then: stdout.size() == 1 - stdout[0] == "Linage graph for lid://12345/file.bam rendered in ${outputHtml}" + stdout[0] == "Rendered lineage graph for lid://12345/file.bam to ${outputHtml}" outputHtml.exists() outputHtml.text == expectedOutput } @@ -300,16 +299,15 @@ class LinCommandImplTest extends Specification{ [new Parameter( "String", "sample_id","ggal_gut"), new Parameter("Integer","reads",2)]) lidFile3.text = encoder.encode(entry) - final network = """flowchart BT - lid://12345/file.bam@{shape: document, label: "lid://12345/file.bam"} - lid://12345@{shape: processes, label: "run_name [lid://12345]"} - ggal_gut@{shape: document, label: "ggal_gut"} - 2.0@{shape: document, label: "2.0"} - - lid://12345 -->lid://12345/file.bam - ggal_gut -->lid://12345 - 2.0 -->lid://12345 -""" + final network = """\ + flowchart TB + lid://12345/file.bam["lid://12345/file.bam"] + lid://12345(["run_name [lid://12345]"]) + ggal_gut["ggal_gut"] + 2.0["2.0"] + lid://12345 --> lid://12345/file.bam + ggal_gut --> lid://12345 + 2.0 --> lid://12345""".stripIndent() final template = MermaidHtmlRenderer.readTemplate() def expectedOutput = template.replace('REPLACE_WITH_NETWORK_DATA', network) @@ -324,7 +322,7 @@ class LinCommandImplTest extends Specification{ then: stdout.size() == 1 - stdout[0] == "Linage graph for lid://12345/file.bam rendered in ${outputHtml}" + stdout[0] == "Rendered lineage graph for lid://12345/file.bam to ${outputHtml}" outputHtml.exists() outputHtml.text == expectedOutput }