Skip to content

Improve lineage diagram #6052

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
May 8, 2025
Merged

Improve lineage diagram #6052

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0edc43e
Improve lineage diagram
bentsherman May 8, 2025
0ac1dbe
Restore support for WorkflowRun
bentsherman May 8, 2025
f8e3ec7
Fix failing tests
bentsherman May 8, 2025
79fc4f2
Merge branch 'master' into lineage-improve-render [ci fast]
pditommaso May 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 14 additions & 15 deletions modules/nextflow/src/test/groovy/nextflow/cli/CmdLineageTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,20 +227,19 @@ class CmdLineageTest extends Specification {
'this is a script',
null,null, null, null, null, [:],[], null)
lidFile5.text = encoder.encode(entry)
final network = """flowchart BT
lid://12345/file.bam@{shape: document, label: "lid://12345/file.bam"}
lid://123987/file.bam@{shape: document, label: "lid://123987/file.bam"}
lid://123987@{shape: process, label: "foo [lid://123987]"}
ggal_gut@{shape: document, label: "ggal_gut"}
lid://45678/output.txt@{shape: document, label: "lid://45678/output.txt"}
lid://45678@{shape: process, label: "bar [lid://45678]"}

lid://123987/file.bam -->lid://12345/file.bam
lid://123987 -->lid://123987/file.bam
ggal_gut -->lid://123987
lid://45678/output.txt -->lid://123987
lid://45678 -->lid://45678/output.txt
"""
final network = """\
flowchart TB
lid://12345/file.bam["lid://12345/file.bam"]
lid://123987/file.bam["lid://123987/file.bam"]
lid://123987(["foo [lid://123987]"])
ggal_gut["ggal_gut"]
lid://45678/output.txt["lid://45678/output.txt"]
lid://45678(["bar [lid://45678]"])
lid://123987/file.bam --> lid://12345/file.bam
lid://123987 --> lid://123987/file.bam
ggal_gut --> lid://123987
lid://45678/output.txt --> lid://123987
lid://45678 --> lid://45678/output.txt""".stripIndent()
final template = MermaidHtmlRenderer.readTemplate()
def expectedOutput = template.replace('REPLACE_WITH_NETWORK_DATA', network)

Expand All @@ -256,7 +255,7 @@ class CmdLineageTest extends Specification {

then:
stdout.size() == 1
stdout[0] == "Linage graph for lid://12345/file.bam rendered in ${outputHtml}"
stdout[0] == "Rendered lineage graph for lid://12345/file.bam to ${outputHtml}"
outputHtml.exists()
outputHtml.text == expectedOutput

Expand Down
144 changes: 3 additions & 141 deletions modules/nf-lineage/src/main/nextflow/lineage/cli/LinCommandImpl.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,21 +21,16 @@ import static nextflow.lineage.fs.LinPath.*
import java.nio.charset.StandardCharsets
import java.nio.file.Path

import groovy.transform.Canonical
import groovy.transform.CompileStatic
import nextflow.Session
import nextflow.cli.CmdLineage
import nextflow.config.ConfigMap
import nextflow.dag.MermaidHtmlRenderer
import nextflow.lineage.LinHistoryRecord
import nextflow.lineage.LinPropertyValidator
import nextflow.lineage.LinStore
import nextflow.lineage.LinStoreFactory
import nextflow.lineage.LinUtils
import nextflow.lineage.model.FileOutput
import nextflow.lineage.model.Parameter
import nextflow.lineage.model.TaskRun
import nextflow.lineage.model.WorkflowRun
import nextflow.lineage.serde.LinEncoder
import nextflow.ui.TableBuilder
import org.eclipse.jgit.diff.DiffAlgorithm
Expand All @@ -52,14 +47,7 @@ class LinCommandImpl implements CmdLineage.LinCommand {

private static final Path DEFAULT_HTML_FILE = Path.of("lineage.html")

@Canonical
static class Edge {
String source
String destination
String label
}

static final private String ERR_NOT_LOADED = 'Error lineage store not loaded - Check Nextflow configuration'
private static final String ERR_NOT_LOADED = 'Error lineage store not loaded - Check Nextflow configuration'

@Override
void list(ConfigMap config) {
Expand Down Expand Up @@ -119,139 +107,13 @@ class LinCommandImpl implements CmdLineage.LinCommand {
}
try {
final renderFile = args.size() > 1 ? Path.of(args[1]) : DEFAULT_HTML_FILE
renderLineage(store, args[0], renderFile)
println("Linage graph for ${args[0]} rendered in $renderFile")
new LinDagRenderer(store).render(args[0], renderFile)
println("Rendered lineage graph for ${args[0]} to $renderFile")
} catch (Throwable e) {
println("ERROR: rendering lineage graph - ${e.message}")
}
}

private void renderLineage(LinStore store, String dataLid, Path file) {
def lines = [] as List<String>
lines << "flowchart BT".toString()
final nodesToRender = new LinkedList<String>()
nodesToRender.add(dataLid)
final edgesToRender = new LinkedList<Edge>()
while (!nodesToRender.isEmpty()) {
final node = nodesToRender.removeFirst()
processNode(lines, node, nodesToRender, edgesToRender, store)
}
lines << ""
edgesToRender.each { lines << " ${it.source} -->${it.destination}".toString() }
lines << ""
lines.join('\n')
final template = MermaidHtmlRenderer.readTemplate()
file.text = template.replace('REPLACE_WITH_NETWORK_DATA', lines.join('\n'))
}

private String safeId( String rawId){
return rawId.replaceAll(/[^a-zA-Z0-9_.:\/\-]/, '_')
}

private void processNode(List<String> lines, String nodeToRender, LinkedList<String> nodes, LinkedList<Edge> edges, LinStore store) {
if (!isLidUri(nodeToRender))
throw new Exception("Identifier is not a lineage URL")
final key = nodeToRender.substring(LID_PROT.size())
final lidObject = store.load(key)
switch (lidObject.getClass()) {
case FileOutput:
processDataOutput(lidObject as FileOutput, lines, nodeToRender, nodes, edges)
break;

case WorkflowRun:
processWorkflowRun(lidObject as WorkflowRun, lines, nodeToRender, edges)
break

case TaskRun:
processTaskRun(lidObject as TaskRun, lines, nodeToRender, nodes, edges)
break

default:
throw new Exception("Unrecognized type reference ${lidObject.getClass().getSimpleName()}")
}
}

private void processTaskRun(TaskRun taskRun, List<String> lines, String nodeToRender, LinkedList<String> nodes, LinkedList<Edge> edges) {
lines << " ${nodeToRender}@{shape: process, label: \"${taskRun.name} [$nodeToRender]\"}".toString()
final parameters = taskRun.input
for (Parameter source : parameters) {
if (source.type.equals("path")) {
manageFileInParam(lines, nodeToRender, nodes, edges, source.value)
} else {
final label = convertToLabel(source.value.toString())
final id = safeId(source.value.toString())
lines << " ${id}@{shape: document, label: \"${label}\"}".toString();
edges.add(new Edge(id, nodeToRender))
}
}
}

private void processWorkflowRun(WorkflowRun wfRun, List<String> lines, String nodeToRender, LinkedList<Edge> edges) {
lines << """ ${nodeToRender}@{shape: processes, label: \"${wfRun.name} [${nodeToRender}]\"}""".toString()
final parameters = wfRun.params
parameters.each {
final label = convertToLabel(it.value.toString())
final id = safeId(it.value.toString())
lines << " ${id}@{shape: document, label: \"${label}\"}".toString();
edges.add(new Edge(id, nodeToRender))
}
}

private void processDataOutput(FileOutput lidObject, List<String> lines, String nodeToRender, LinkedList<String> nodes, LinkedList<Edge> edges){
lines << " ${nodeToRender}@{shape: document, label: \"${nodeToRender}\"}".toString();
final source = lidObject.source
if(! source )
return
if (isLidUri(source)) {
nodes.add(source)
edges.add(new Edge(source, nodeToRender))
} else {
final label = convertToLabel(source)
final id = safeId(source)
lines << " ${id}@{shape: document, label: \"${label}\"}".toString();
edges.add(new Edge(id, nodeToRender))
}
}

private String convertToLabel(String label){
return label.replace('http', 'h\u200Ettp')
}

private void manageFileInParam(List<String> lines, String nodeToRender, LinkedList<String> nodes, LinkedList<Edge> edges, value){
if (value instanceof Collection) {
value.each { manageFileInParam(lines, nodeToRender, nodes, edges, it) }
return
}
if (value instanceof CharSequence) {
final source = value.toString()
if (isLidUri(source)) {
nodes.add(source)
edges.add(new Edge(source, nodeToRender))
return
}
}
if (value instanceof Map ) {
if (value.path) {
final path = value.path.toString()
if (isLidUri(path)) {
nodes.add(path)
edges.add(new Edge(path, nodeToRender))
return
} else {
final label = convertToLabel(path)
final id = safeId(path)
lines << " ${id}@{shape: document, label: \"${label}\"}".toString();
edges.add(new Edge(id, nodeToRender))
return
}
}
}
final label = convertToLabel(value.toString())
final id = safeId(value.toString())
lines << " ${id}@{shape: document, label: \"${label}\"}".toString();
edges.add(new Edge(id, nodeToRender))
}

@Override
void diff(ConfigMap config, List<String> args) {
if (!isLidUri(args[0]) || !isLidUri(args[1]))
Expand Down
Loading
Loading