From e09529b93ff93d08d27d29b09158e73bba16f3f3 Mon Sep 17 00:00:00 2001 From: bogdang989 Date: Tue, 26 Jun 2018 16:27:17 +0200 Subject: [PATCH 1/4] File metadata property added --- CommandLineTool.yml | 8 ++++++++ Process.yml | 5 ++++- tests/inherit-metadata-job.yml | 5 +++++ tests/inherit-metadata.cwl | 26 ++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 tests/inherit-metadata-job.yml create mode 100644 tests/inherit-metadata.cwl diff --git a/CommandLineTool.yml b/CommandLineTool.yml index 7e267dd..a453cc2 100644 --- a/CommandLineTool.yml +++ b/CommandLineTool.yml @@ -251,6 +251,14 @@ $graph: array of a single element. Additionally, if `loadContents` is `true`, the File objects must include up to the first 64 KiB of file contents in the `contents` field. + - name: addMetadata + type: + - "null" + - string + - Expression + doc: | + Evaluate an expression to generate the output file metadata. Expression + should return a key-value map. - name: CommandLineBindable type: record diff --git a/Process.yml b/Process.yml index 4d2aafb..7926519 100644 --- a/Process.yml +++ b/Process.yml @@ -279,7 +279,10 @@ $graph: If `loadContents` of `inputBinding` or `outputBinding` is true and `location` is valid, the implementation must read up to the first 64 KiB of text from the file and place it in the "contents" field. - + - name: metadata + type: Any? + doc: | + Metadata further describing the file. - name: Directory type: record diff --git a/tests/inherit-metadata-job.yml b/tests/inherit-metadata-job.yml new file mode 100644 index 0000000..93977f9 --- /dev/null +++ b/tests/inherit-metadata-job.yml @@ -0,0 +1,5 @@ +input: + class: File + path: hello.txt + metadata: + hello: world \ No newline at end of file diff --git a/tests/inherit-metadata.cwl b/tests/inherit-metadata.cwl new file mode 100644 index 0000000..6c4584b --- /dev/null +++ b/tests/inherit-metadata.cwl @@ -0,0 +1,26 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.1.0-dev1 +baseCommand: + - echo + - this + - file + - has + - metadata +inputs: + - id: input + type: File? +outputs: + - id: out + type: File + secondaryFiles: + - $(self.basename) + outputBinding: + glob: out + addMetadata: $(inputs.input.metadata) +requirements: + - class: DockerRequirement + dockerPull: 'ubuntu:16.04' + - class: InlineJavascriptRequirement +stdout: out \ No newline at end of file From c864b88e44d1b03f2b346d1764c890be245b30c4 Mon Sep 17 00:00:00 2001 From: bogdang989 Date: Mon, 20 Aug 2018 15:38:19 +0200 Subject: [PATCH 2/4] Additional addMetadata description --- CommandLineTool.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CommandLineTool.yml b/CommandLineTool.yml index a453cc2..f72693e 100644 --- a/CommandLineTool.yml +++ b/CommandLineTool.yml @@ -223,6 +223,7 @@ $graph: - loadContents - outputEval - secondaryFiles + - addMetadata fields: - name: glob type: @@ -258,7 +259,9 @@ $graph: - Expression doc: | Evaluate an expression to generate the output file metadata. Expression - should return a key-value map. + should return a key-value map. Returned map should be set as the + `metadata` property of the output file or files. The same map + should be set as the `metadata` property of all secondary files. - name: CommandLineBindable type: record From 31afe3559b6142838946106617e46b8310fcac49 Mon Sep 17 00:00:00 2001 From: bogdang989 Date: Mon, 20 Aug 2018 15:51:06 +0200 Subject: [PATCH 3/4] Additional metadata description --- Process.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Process.yml b/Process.yml index 7926519..a8db0e4 100644 --- a/Process.yml +++ b/Process.yml @@ -282,7 +282,12 @@ $graph: - name: metadata type: Any? doc: | - Metadata further describing the file. + Metadata should be a key-value map further describing a `File`. + Metadata can be propagated from inputs to outputs of a tool using + `addMetadata` option in the `CommandOutputBinding`. It is up to the + implementation to make use of the `metadata` property of final outputs + of the execution. + - name: Directory type: record From 5bbc3195fde64605a3d274ef7a4211dc809994d2 Mon Sep 17 00:00:00 2001 From: bogdang989 Date: Tue, 11 Sep 2018 13:36:58 +0200 Subject: [PATCH 4/4] Allow input reference in addMetadata --- CommandLineTool.yml | 15 ++++++--- conformance_tests.yaml | 58 ++++++++++++++++++++++++++++++++++ tests/inherit-metadata-job.yml | 11 +++++-- tests/inherit-metadata.cwl | 19 ++++++++--- 4 files changed, 93 insertions(+), 10 deletions(-) diff --git a/CommandLineTool.yml b/CommandLineTool.yml index f72693e..6ec6554 100644 --- a/CommandLineTool.yml +++ b/CommandLineTool.yml @@ -257,11 +257,18 @@ $graph: - "null" - string - Expression + - string[] doc: | - Evaluate an expression to generate the output file metadata. Expression - should return a key-value map. Returned map should be set as the - `metadata` property of the output file or files. The same map - should be set as the `metadata` property of all secondary files. + If the value is an expression, expression should return a key-value map. + Returned map should be set as the `metadata` property of the output file + or files. The same map should be set as the `metadata` property of all + secondary files. + + If the value is a string or an array of strings, these must be input + parameter references. The `metadata` fields for the referenced inputs are + intersected and the resulting key-value map is set as the `metadata` + property of the output file or files. The same map should be set as the + `metadata` property of all secondary files. - name: CommandLineBindable type: record diff --git a/conformance_tests.yaml b/conformance_tests.yaml index a5d3e65..4ada7c4 100644 --- a/conformance_tests.yaml +++ b/conformance_tests.yaml @@ -1768,3 +1768,61 @@ tool: tests/networkaccess2.cwl doc: Test networkaccess is disabled by default tags: [ networkaccess ] + + + +- job: tests/inherit-metadata-job.yml + output: { + "out": { + "basename": "out", + "checksum": "sha1$4a7edca069beb3fbdcc3ce9d8626025d262c046a", + "class": "File", + "location": "out", + "metadata": { + "foo": 1, + "hello": "world" + }, + "secondaryFiles": [ + { + "basename": "out", + "checksum": "sha1$4a7edca069beb3fbdcc3ce9d8626025d262c046a", + "class": "File", + "location": "out", + "metadata": { + "foo": 1, + "hello": "world" + }, + "size": 23 + } + ], + "size": 23 + }, + "out_intersected": [ + { + "basename": "out", + "checksum": "sha1$4a7edca069beb3fbdcc3ce9d8626025d262c046a", + "class": "File", + "location": "out", + "metadata": { + "hello": "world" + }, + "secondaryFiles": [ + { + "basename": "out", + "checksum": "sha1$4a7edca069beb3fbdcc3ce9d8626025d262c046a", + "class": "File", + "location": "out", + "metadata": { + "hello": "world" + }, + "size": 23 + } + ], + "size": 23 + } + ] + } + tool: tests/inherit-metadata.cwl + doc: Test addmetadata via expression or input id + tags: [ addmetadata ] + diff --git a/tests/inherit-metadata-job.yml b/tests/inherit-metadata-job.yml index 93977f9..df7a300 100644 --- a/tests/inherit-metadata-job.yml +++ b/tests/inherit-metadata-job.yml @@ -1,5 +1,12 @@ -input: +input_file: class: File path: hello.txt metadata: - hello: world \ No newline at end of file + hello: world + foo: 1 +input_list: + - class: File + path: whale.txt + metadata: + hello: world + foo: 2 \ No newline at end of file diff --git a/tests/inherit-metadata.cwl b/tests/inherit-metadata.cwl index 6c4584b..212dd78 100644 --- a/tests/inherit-metadata.cwl +++ b/tests/inherit-metadata.cwl @@ -9,8 +9,10 @@ baseCommand: - has - metadata inputs: - - id: input - type: File? + - id: input_file + type: File + - id: input_list + type: File[] outputs: - id: out type: File @@ -18,9 +20,18 @@ outputs: - $(self.basename) outputBinding: glob: out - addMetadata: $(inputs.input.metadata) + addMetadata: $(inputs.input_file.metadata) + - id: out_intersected + type: File[] + secondaryFiles: + - $(self.basename) + outputBinding: + glob: out + addMetadata: + - input_file + - input_list requirements: - class: DockerRequirement - dockerPull: 'ubuntu:16.04' + dockerPull: 'debian:stretch-slim' - class: InlineJavascriptRequirement stdout: out \ No newline at end of file