Skip to content

Commit d1cae3b

Browse files
pditommasopditommaso
committed
Store lineage meta in root location and default to .lineage path [ci fast]
Signed-off-by: Paolo Di Tommaso <[email protected]>
1 parent 77736b5 commit d1cae3b

File tree

7 files changed

+67
-86
lines changed

7 files changed

+67
-86
lines changed

‎modules/nextflow/src/test/groovy/nextflow/cli/CmdLineageTest.groovy

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ class CmdLineageTest extends Specification {
6767
def folder = Files.createTempDirectory('test').toAbsolutePath()
6868
def configFile = folder.resolve('nextflow.config')
6969
configFile.text = "lineage.enabled = true\nlineage.store.location = '$folder'".toString()
70-
def historyFile = folder.resolve(".meta/.history")
70+
def historyFile = folder.resolve(".history")
7171
def lidLog = new DefaultLinHistoryLog(historyFile)
7272
def uniqueId = UUID.randomUUID()
7373
def date = new Date();
@@ -99,7 +99,7 @@ class CmdLineageTest extends Specification {
9999
def folder = Files.createTempDirectory('test').toAbsolutePath()
100100
def configFile = folder.resolve('nextflow.config')
101101
configFile.text = "lineage.enabled = true\nlineage.store.location = '$folder'".toString()
102-
def historyFile = folder.resolve(".meta/.history")
102+
def historyFile = folder.resolve(".history")
103103
Files.createDirectories(historyFile.parent)
104104
def launcher = Mock(Launcher){
105105
getOptions() >> new CliOptions(config: [configFile.toString()])
@@ -128,7 +128,7 @@ class CmdLineageTest extends Specification {
128128
def folder = Files.createTempDirectory('test').toAbsolutePath()
129129
def configFile = folder.resolve('nextflow.config')
130130
configFile.text = "lineage.enabled = true\nlineage.store.location = '$folder'".toString()
131-
def lidFile = folder.resolve(".meta/12345/.data.json")
131+
def lidFile = folder.resolve("12345/.data.json")
132132
Files.createDirectories(lidFile.parent)
133133
def launcher = Mock(Launcher){
134134
getOptions() >> new CliOptions(config: [configFile.toString()])
@@ -194,11 +194,11 @@ class CmdLineageTest extends Specification {
194194
def launcher = Mock(Launcher){
195195
getOptions() >> new CliOptions(config: [configFile.toString()])
196196
}
197-
def lidFile = folder.resolve(".meta/12345/file.bam/.data.json")
198-
def lidFile2 = folder.resolve(".meta/123987/file.bam/.data.json")
199-
def lidFile3 = folder.resolve(".meta/123987/.data.json")
200-
def lidFile4 = folder.resolve(".meta/45678/output.txt/.data.json")
201-
def lidFile5 = folder.resolve(".meta/45678/.data.json")
197+
def lidFile = folder.resolve("12345/file.bam/.data.json")
198+
def lidFile2 = folder.resolve("123987/file.bam/.data.json")
199+
def lidFile3 = folder.resolve("123987/.data.json")
200+
def lidFile4 = folder.resolve("45678/output.txt/.data.json")
201+
def lidFile5 = folder.resolve("45678/.data.json")
202202
Files.createDirectories(lidFile.parent)
203203
Files.createDirectories(lidFile2.parent)
204204
Files.createDirectories(lidFile3.parent)
@@ -270,7 +270,7 @@ class CmdLineageTest extends Specification {
270270
def folder = Files.createTempDirectory('test').toAbsolutePath()
271271
def configFile = folder.resolve('nextflow.config')
272272
configFile.text = "lineage.enabled = true\nlineage.store.location = '$folder'".toString()
273-
def lidFile = folder.resolve(".meta/12345/.data.json")
273+
def lidFile = folder.resolve("12345/.data.json")
274274
Files.createDirectories(lidFile.parent)
275275
def launcher = Mock(Launcher){
276276
getOptions() >> new CliOptions(config: [configFile.toString()])
@@ -305,7 +305,7 @@ class CmdLineageTest extends Specification {
305305
def folder = Files.createTempDirectory('test').toAbsolutePath()
306306
def configFile = folder.resolve('nextflow.config')
307307
configFile.text = "lineage.enabled = true\nlineage.store.location = '$folder'".toString()
308-
def lidFile = folder.resolve(".meta/12345/.data.json")
308+
def lidFile = folder.resolve("12345/.data.json")
309309
Files.createDirectories(lidFile.parent)
310310
def launcher = Mock(Launcher){
311311
getOptions() >> new CliOptions(config: [configFile.toString()])

‎modules/nf-lineage/src/main/nextflow/lineage/DefaultLinStore.groovy

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,11 @@ import java.nio.file.attribute.BasicFileAttributes
2424

2525
import groovy.transform.CompileStatic
2626
import groovy.util.logging.Slf4j
27-
import nextflow.lineage.serde.LinEncoder
28-
import nextflow.lineage.serde.LinSerializable
29-
import nextflow.lineage.config.LineageConfig
3027
import nextflow.exception.AbortOperationException
3128
import nextflow.file.FileHelper
32-
import nextflow.util.TestOnly
33-
29+
import nextflow.lineage.config.LineageConfig
30+
import nextflow.lineage.serde.LinEncoder
31+
import nextflow.lineage.serde.LinSerializable
3432
/**
3533
* Default Implementation for the a lineage store.
3634
*
@@ -42,22 +40,19 @@ class DefaultLinStore implements LinStore {
4240

4341
private static String HISTORY_FILE_NAME = ".history"
4442
private static final String METADATA_FILE = '.data.json'
45-
private static final String METADATA_PATH = '.meta'
46-
private static final String DEFAULT_LOCATION = 'lineage'
43+
private static final String DEFAULT_LOCATION = '.lineage'
4744

48-
private Path metaLocation
4945
private Path location
5046
private LinHistoryLog historyLog
5147
private LinEncoder encoder
5248

5349
DefaultLinStore open(LineageConfig config) {
5450
location = toLocationPath(config.store.location)
55-
metaLocation = location.resolve(METADATA_PATH)
5651
encoder = new LinEncoder()
57-
if( !Files.exists(metaLocation) && !Files.createDirectories(metaLocation) ) {
58-
throw new AbortOperationException("Unable to create lineage store directory: $metaLocation")
52+
if( !Files.exists(location) && !Files.createDirectories(location) ) {
53+
throw new AbortOperationException("Unable to create lineage store directory: $location")
5954
}
60-
historyLog = new DefaultLinHistoryLog(metaLocation.resolve(HISTORY_FILE_NAME))
55+
historyLog = new DefaultLinHistoryLog(location.resolve(HISTORY_FILE_NAME))
6156
return this
6257
}
6358

@@ -69,15 +64,15 @@ class DefaultLinStore implements LinStore {
6964

7065
@Override
7166
void save(String key, LinSerializable value) {
72-
final path = metaLocation.resolve("$key/$METADATA_FILE")
67+
final path = location.resolve("$key/$METADATA_FILE")
7368
Files.createDirectories(path.parent)
7469
log.debug "Save LID file path: $path"
7570
path.text = encoder.encode(value)
7671
}
7772

7873
@Override
7974
LinSerializable load(String key) {
80-
final path = metaLocation.resolve("$key/$METADATA_FILE")
75+
final path = location.resolve("$key/$METADATA_FILE")
8176
log.debug("Loading from path $path")
8277
if (path.exists())
8378
return encoder.decode(path.text) as LinSerializable
@@ -89,11 +84,6 @@ class DefaultLinStore implements LinStore {
8984
return location
9085
}
9186

92-
@TestOnly
93-
Path getMetadataPath() {
94-
return metaLocation
95-
}
96-
9787
@Override
9888
LinHistoryLog getHistoryLog(){
9989
return historyLog
@@ -114,7 +104,7 @@ class DefaultLinStore implements LinStore {
114104
private Map<String, LinSerializable> searchAllFiles(Map<String,String> params) {
115105
final results = new HashMap<String, LinSerializable>()
116106

117-
Files.walkFileTree(metaLocation, new FileVisitor<Path>() {
107+
Files.walkFileTree(location, new FileVisitor<Path>() {
118108

119109
@Override
120110
FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
@@ -126,7 +116,7 @@ class DefaultLinStore implements LinStore {
126116
if (file.name.startsWith('.data.json') ) {
127117
final lidObject = encoder.decode(file.text)
128118
if (LinUtils.checkParams(lidObject, params)){
129-
results.put(metaLocation.relativize(file.getParent()).toString(), lidObject as LinSerializable)
119+
results.put(location.relativize(file.getParent()).toString(), lidObject as LinSerializable)
130120
}
131121
}
132122
FileVisitResult.CONTINUE

‎modules/nf-lineage/src/test/nextflow/lineage/DefaultLinStoreTest.groovy

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,12 +45,10 @@ class DefaultLinStoreTest extends Specification {
4545
Path tempDir
4646

4747
Path storeLocation
48-
Path metaLocation
4948
LineageConfig config
5049

5150
def setup() {
5251
storeLocation = tempDir.resolve("store")
53-
metaLocation = storeLocation.resolve(".meta")
5452
def configMap = [enabled: true, store: [location: storeLocation.toString(), logLocation: storeLocation.resolve(".log").toString()]]
5553
config = new LineageConfig(configMap)
5654
}
@@ -62,7 +60,7 @@ class DefaultLinStoreTest extends Specification {
6260
store.open(config)
6361
def historyLog = store.getHistoryLog()
6462
then:
65-
store.getMetadataPath() == metaLocation
63+
store.getLocation() == storeLocation
6664
historyLog != null
6765
historyLog instanceof DefaultLinHistoryLog
6866
}
@@ -78,7 +76,7 @@ class DefaultLinStoreTest extends Specification {
7876
lidStore.save(key, value)
7977

8078
then:
81-
def filePath = metaLocation.resolve("$key/.data.json")
79+
def filePath = storeLocation.resolve("$key/.data.json")
8280
Files.exists(filePath)
8381
filePath.text == new LinEncoder().encode(value)
8482
}

‎modules/nf-lineage/src/test/nextflow/lineage/LinObserverTest.groovy

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ class LinObserverTest extends Specification {
185185
observer.onFlowCreate(session)
186186
observer.onFlowBegin()
187187
then:
188-
folder.resolve(".meta/${observer.executionHash}/.data.json").text == new LinEncoder().encode(workflowRun)
188+
folder.resolve("${observer.executionHash}/.data.json").text == new LinEncoder().encode(workflowRun)
189189

190190
cleanup:
191191
folder?.deleteDir()
@@ -370,7 +370,7 @@ class LinObserverTest extends Specification {
370370
when:
371371
observer.storeTaskOutput(task, outFile)
372372
then:
373-
folder.resolve(".meta/${hash}/foo/bar/file.bam/.data.json").text == new LinEncoder().encode(output)
373+
folder.resolve("${hash}/foo/bar/file.bam/.data.json").text == new LinEncoder().encode(output)
374374

375375
cleanup:
376376
folder?.deleteDir()
@@ -533,7 +533,7 @@ class LinObserverTest extends Specification {
533533
def output1 = new FileOutput(outFile1.toString(), new Checksum(fileHash1, "nextflow", "standard"),
534534
"lid://123987/file.bam", "$LID_PROT${observer.executionHash}", null,
535535
attrs1.size(), LinUtils.toDate(attrs1.creationTime()), LinUtils.toDate(attrs1.lastModifiedTime()) )
536-
folder.resolve(".meta/${observer.executionHash}/foo/file.bam/.data.json").text == encoder.encode(output1)
536+
folder.resolve("${observer.executionHash}/foo/file.bam/.data.json").text == encoder.encode(output1)
537537

538538
when: 'publish without source path'
539539
def outFile2 = outputDir.resolve('foo/file2.bam')
@@ -547,7 +547,7 @@ class LinObserverTest extends Specification {
547547
def output2 = new FileOutput(outFile2.toString(), new Checksum(fileHash2, "nextflow", "standard"),
548548
"lid://${observer.executionHash}" , "lid://${observer.executionHash}", null,
549549
attrs2.size(), LinUtils.toDate(attrs2.creationTime()), LinUtils.toDate(attrs2.lastModifiedTime()) )
550-
folder.resolve(".meta/${observer.executionHash}/foo/file2.bam/.data.json").text == encoder.encode(output2)
550+
folder.resolve("${observer.executionHash}/foo/file2.bam/.data.json").text == encoder.encode(output2)
551551

552552
when: 'Workflow complete'
553553
observer.onFlowComplete()

‎modules/nf-lineage/src/test/nextflow/lineage/cli/LinCommandImplTest.groovy

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ class LinCommandImplTest extends Specification{
7373

7474
def 'should print executions lids' (){
7575
given:
76-
def historyFile = storeLocation.resolve(".meta/.history")
76+
def historyFile = storeLocation.resolve(".history")
7777
def lidLog = new DefaultLinHistoryLog(historyFile)
7878
def uniqueId = UUID.randomUUID()
7979
def date = new Date();
@@ -115,7 +115,7 @@ class LinCommandImplTest extends Specification{
115115

116116
def 'should show lid content' (){
117117
given:
118-
def lidFile = storeLocation.resolve(".meta/12345/.data.json")
118+
def lidFile = storeLocation.resolve("12345/.data.json")
119119
Files.createDirectories(lidFile.parent)
120120
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
121121
def encoder = new LinEncoder().withPrettyPrint(true)
@@ -160,11 +160,11 @@ class LinCommandImplTest extends Specification{
160160

161161
def outputHtml = tmpDir.resolve('lineage.html')
162162

163-
def lidFile = storeLocation.resolve(".meta/12345/file.bam/.data.json")
164-
def lidFile2 = storeLocation.resolve(".meta/123987/file.bam/.data.json")
165-
def lidFile3 = storeLocation.resolve(".meta/123987/.data.json")
166-
def lidFile4 = storeLocation.resolve(".meta/45678/output.txt/.data.json")
167-
def lidFile5 = storeLocation.resolve(".meta/45678/.data.json")
163+
def lidFile = storeLocation.resolve("12345/file.bam/.data.json")
164+
def lidFile2 = storeLocation.resolve("123987/file.bam/.data.json")
165+
def lidFile3 = storeLocation.resolve("123987/.data.json")
166+
def lidFile4 = storeLocation.resolve("45678/output.txt/.data.json")
167+
def lidFile5 = storeLocation.resolve("45678/.data.json")
168168
Files.createDirectories(lidFile.parent)
169169
Files.createDirectories(lidFile2.parent)
170170
Files.createDirectories(lidFile3.parent)
@@ -235,8 +235,8 @@ class LinCommandImplTest extends Specification{
235235

236236
def outputHtml = tmpDir.resolve('lineage.html')
237237

238-
def lidFile = storeLocation.resolve(".meta/12345/file.bam/.data.json")
239-
def lidFile3 = storeLocation.resolve(".meta/12345/.data.json")
238+
def lidFile = storeLocation.resolve("12345/file.bam/.data.json")
239+
def lidFile3 = storeLocation.resolve("12345/.data.json")
240240
Files.createDirectories(lidFile.parent)
241241
Files.createDirectories(lidFile3.parent)
242242
def encoder = new LinEncoder()
@@ -280,7 +280,7 @@ class LinCommandImplTest extends Specification{
280280

281281
def 'should show query results'(){
282282
given:
283-
def lidFile = storeLocation.resolve(".meta/12345/.data.json")
283+
def lidFile = storeLocation.resolve("12345/.data.json")
284284
Files.createDirectories(lidFile.parent)
285285
def encoder = new LinEncoder().withPrettyPrint(true)
286286
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
@@ -305,9 +305,9 @@ class LinCommandImplTest extends Specification{
305305

306306
def 'should show query with fragment'(){
307307
given:
308-
def lidFile = storeLocation.resolve(".meta/12345/.data.json")
308+
def lidFile = storeLocation.resolve("12345/.data.json")
309309
Files.createDirectories(lidFile.parent)
310-
def lidFile2 = storeLocation.resolve(".meta/67890/.data.json")
310+
def lidFile2 = storeLocation.resolve("67890/.data.json")
311311
Files.createDirectories(lidFile2.parent)
312312
def encoder = new LinEncoder().withPrettyPrint(true)
313313
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
@@ -334,9 +334,9 @@ class LinCommandImplTest extends Specification{
334334

335335
def 'should diff'(){
336336
given:
337-
def lidFile = storeLocation.resolve(".meta/12345/.data.json")
337+
def lidFile = storeLocation.resolve("12345/.data.json")
338338
Files.createDirectories(lidFile.parent)
339-
def lidFile2 = storeLocation.resolve(".meta/67890/.data.json")
339+
def lidFile2 = storeLocation.resolve("67890/.data.json")
340340
Files.createDirectories(lidFile2.parent)
341341
def encoder = new LinEncoder().withPrettyPrint(true)
342342
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
@@ -386,7 +386,7 @@ class LinCommandImplTest extends Specification{
386386

387387
def 'should print error if no entry found diff'(){
388388
given:
389-
def lidFile = storeLocation.resolve(".meta/12345/.data.json")
389+
def lidFile = storeLocation.resolve("12345/.data.json")
390390
Files.createDirectories(lidFile.parent)
391391
def encoder = new LinEncoder().withPrettyPrint(true)
392392
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)
@@ -435,9 +435,9 @@ class LinCommandImplTest extends Specification{
435435

436436
def 'should find metadata descriptions'(){
437437
given:
438-
def lidFile = storeLocation.resolve(".meta/123987/file.bam/.data.json")
438+
def lidFile = storeLocation.resolve("123987/file.bam/.data.json")
439439
Files.createDirectories(lidFile.parent)
440-
def lidFile2 = storeLocation.resolve(".meta/123987/file2.bam/.data.json")
440+
def lidFile2 = storeLocation.resolve("123987/file2.bam/.data.json")
441441
Files.createDirectories(lidFile2.parent)
442442
def encoder = new LinEncoder().withPrettyPrint(true)
443443
def time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(123456789), ZoneOffset.UTC)

0 commit comments

Comments
 (0)