Skip to content

Commit c13e26a

Browse files
mr-cmr-c
authored
Merge branch 'main' into input_annotations
2 parents 45ddd8d + d2059c7 commit c13e26a

39 files changed

+385
-141
lines changed

‎.github/workflows/ci-tests.yml

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ jobs:
7777
key: mypy-${{ env.py-semver }}
7878

7979
- name: Test with tox
80-
run: tox
80+
run: APPTAINER_TMPDIR=${RUNNER_TEMP} tox
8181

8282
- name: Upload coverage to Codecov
8383
if: ${{ matrix.step == 'unit' }}
@@ -156,7 +156,7 @@ jobs:
156156
chmod a-w .
157157
158158
- name: run tests
159-
run: make test
159+
run: APPTAINER_TMPDIR=${RUNNER_TEMP} make test
160160

161161

162162
conformance_tests:
@@ -203,9 +203,15 @@ jobs:
203203
env:
204204
VERSION: ${{ matrix.cwl-version }}
205205
CONTAINER: ${{ matrix.container }}
206-
GIT_TARGET: main
207-
CWLTOOL_OPTIONS: ${{ matrix.extras }}
206+
GIT_TARGET: ${{ matrix.cwl-version == 'v1.2' && '1.2.1_proposed' || 'main' }}
207+
CWLTOOL_OPTIONS: ${{ matrix.cwl-version == 'v1.2' && '--relax-path-checks' || '' }} ${{ matrix.extras }}
208208
run: ./conformance-test.sh
209+
- name: Archive test results
210+
uses: actions/upload-artifact@v3
211+
with:
212+
name: cwl-${{ matrix.cwl-version }}-${{ matrix.container }}${{ matrix.extras }}-conformance-results
213+
path: |
214+
**/cwltool_conf*.xml
209215
- name: Upload coverage to Codecov
210216
uses: codecov/codecov-action@v3
211217
with:

‎.github/workflows/quay-publish.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
- name: Get image tags
1515
id: image_tags
1616
run: |
17-
echo -n "IMAGE_TAGS=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT
17+
echo -n "IMAGE_TAGS=${GITHUB_REF#refs/*/}" >> "$GITHUB_OUTPUT"
1818
- name: record cwltool version
1919
run: |
2020
pip install "setuptools>=61"

‎conformance-test.sh

Lines changed: 37 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ set -x
4141
# The directory where this script resides
4242
SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
4343

44+
TMP_DIR=${TMP_DIR:-${SCRIPT_DIRECTORY}}
45+
4446
# Download archive from GitHub
4547
if [[ "${VERSION}" = "v1.0" ]] ; then
4648
REPO=common-workflow-language
@@ -49,38 +51,40 @@ else
4951
REPO=cwl-$(echo "$VERSION" | sed 's/\(v[0-9]*\.\)\([0-9]*\).*/\1\2/')
5052
fi
5153

52-
if [ ! -d "${REPO}-${GIT_TARGET}" ] ; then
53-
if [ ! -f "${GIT_TARGET}.tar.gz" ] ; then
54-
wget "https://github.com/common-workflow-language/${REPO}/archive/${GIT_TARGET}.tar.gz"
54+
if [ ! -d "${TMP_DIR}/${REPO}-${GIT_TARGET}" ] ; then
55+
if [ ! -f "${TMP_DIR}/${GIT_TARGET}.tar.gz" ] ; then
56+
wget --directory-prefix "${TMP_DIR}" "https://github.com/common-workflow-language/${REPO}/archive/${GIT_TARGET}.tar.gz"
5557
fi
56-
tar xzf "${GIT_TARGET}.tar.gz"
58+
tar --directory "${TMP_DIR}" -x -f "${TMP_DIR}/${GIT_TARGET}.tar.gz"
5759
fi
5860

59-
if [ "${CONTAINER}" == "docker" ]; then
61+
if [ -v SKIP_PULL ] ; then
62+
echo Skipping node container pull.
63+
elif [ "${CONTAINER}" == "docker" ]; then
6064
docker pull docker.io/node:slim
61-
fi
62-
63-
if [ "${CONTAINER}" == "podman" ]; then
65+
elif [ "${CONTAINER}" == "podman" ]; then
6466
podman pull docker.io/node:slim
65-
fi
66-
67-
if [ "${CONTAINER}" == "singularity" ]; then
67+
elif [ "${CONTAINER}" == "singularity" ]; then
6868
export CWL_SINGULARITY_CACHE="$SCRIPT_DIRECTORY/sifcache"
6969
mkdir --parents "${CWL_SINGULARITY_CACHE}"
7070
fi
7171

7272
# Setup environment
73-
venv cwl-conformance-venv
74-
pip install -U setuptools wheel pip
75-
pip uninstall -y cwltool
76-
pip install "${SCRIPT_DIRECTORY}" -r"${SCRIPT_DIRECTORY}/requirements.txt"
77-
pip install 'cwltest>=2.3' pytest-cov pytest-xdist
73+
if [ -v SKIP_INSTALL ] ; then
74+
echo 'Skip installing dependencies; cwltool & cwltest must already be installed'
75+
else
76+
venv "${TMP_DIR}/cwl-conformance-venv"
77+
pip install -U setuptools wheel pip
78+
pip uninstall -y cwltool
79+
pip install "${SCRIPT_DIRECTORY}" -r"${SCRIPT_DIRECTORY}/requirements.txt"
80+
pip install 'cwltest>=2.3' pytest-cov pytest-xdist
81+
fi
7882

7983
# Set conformance test filename
8084
if [[ "${VERSION}" = "v1.0" ]] ; then
81-
CONFORMANCE_TEST="${SCRIPT_DIRECTORY}/${REPO}-${GIT_TARGET}/${VERSION}/conformance_test_v1.0.yaml"
85+
CONFORMANCE_TEST="${TMP_DIR}/${REPO}-${GIT_TARGET}/${VERSION}/conformance_test_v1.0.yaml"
8286
else
83-
CONFORMANCE_TEST="${SCRIPT_DIRECTORY}/${REPO}-${GIT_TARGET}/conformance_tests.yaml"
87+
CONFORMANCE_TEST="${TMP_DIR}/${REPO}-${GIT_TARGET}/conformance_tests.yaml"
8488
fi
8589
cp "${CONFORMANCE_TEST}" "${CONFORMANCE_TEST%".yaml"}.cwltest.yaml"
8690
CONFORMANCE_TEST="${CONFORMANCE_TEST%".yaml"}.cwltest.yaml"
@@ -116,14 +120,18 @@ if (( "${#exclusions[*]}" > 0 )); then
116120
fi
117121

118122
# Build command
119-
TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n auto -rs --junit-xml=${SCRIPT_DIRECTORY}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)"
123+
TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n auto -rs --junit-xml=${TMP_DIR}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)"
120124
if [[ -n "${EXCLUDE}" ]] ; then
121125
TEST_COMMAND="${TEST_COMMAND} --cwl-exclude ${EXCLUDE}"
122126
fi
123-
TEST_COMMAND="${TEST_COMMAND} --cov --cov-config ${SCRIPT_DIRECTORY}/.coveragerc --cov-report= ${PYTEST_EXTRA}"
127+
if [ -v SKIP_COV ] ; then
128+
echo Skipping gathering of coverage information
129+
else
130+
TEST_COMMAND="${TEST_COMMAND} --cov --cov-config ${SCRIPT_DIRECTORY}/.coveragerc --cov-report= ${PYTEST_EXTRA}"
131+
# Clean up all old coverage data
132+
find "${SCRIPT_DIRECTORY}" \( -type f -name .coverage -or -name '.coverage.*' -or -name coverage.xml \) -delete
133+
fi
124134

125-
# Clean up all old coverage data
126-
find "${SCRIPT_DIRECTORY}" \( -type f -name .coverage -or -name '.coverage.*' -or -name coverage.xml \) -delete
127135

128136
if [ "$GIT_BRANCH" = "origin/main" ] && [[ "$VERSION" = "v1.0" ]] && [[ "$CONTAINER" = "docker" ]]
129137
then
@@ -153,11 +161,11 @@ echo CWLTOOL_OPTIONS="${CWLTOOL_OPTIONS}"
153161

154162
# Run test
155163
cp "${SCRIPT_DIRECTORY}/tests/cwl-conformance/cwltool-conftest.py" "$(dirname "${CONFORMANCE_TEST}")/conftest.py"
156-
bash -c "${TEST_COMMAND}"
164+
bash -c "cd ${TMP_DIR} && ${TEST_COMMAND}"
157165
RETURN_CODE=$?
158166

159167
# Coverage report
160-
if [ "${RETURN_CODE}" -eq "0" ] ; then
168+
if [ ! -v SKIP_COV ] && [ "${RETURN_CODE}" -eq "0" ] ; then
161169
coverage report
162170
coverage xml
163171
fi
@@ -172,7 +180,11 @@ then
172180
fi
173181

174182
# Cleanup
175-
deactivate
183+
if [ -z "$SKIP_INSTALL" ] ; then
184+
echo Skipping venv cleanup
185+
else
186+
deactivate
187+
fi
176188
#rm -rf "${GIT_TARGET}.tar.gz" "${SCRIPT_DIRECTORY}/${REPO}-${GIT_TARGET}" "${SCRIPT_DIRECTORY}/cwl-conformance-venv"
177189

178190
# Exit

‎cwltool/argparser.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -378,6 +378,7 @@ def arg_parser() -> argparse.ArgumentParser:
378378

379379
volumegroup = parser.add_mutually_exclusive_group()
380380
volumegroup.add_argument("--verbose", action="store_true", help="Default logging")
381+
volumegroup.add_argument("--no-warnings", action="store_true", help="Only print errors.")
381382
volumegroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.")
382383
volumegroup.add_argument("--debug", action="store_true", help="Print even more logging")
383384

‎cwltool/checker.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
from .errors import WorkflowException
2222
from .loghandler import _logger
2323
from .process import shortname
24-
from .utils import CWLObjectType, CWLOutputAtomType, CWLOutputType, SinkType, aslist
24+
from .utils import CWLObjectType, CWLOutputType, SinkType, aslist
2525

2626

2727
def _get_type(tp):
@@ -90,8 +90,8 @@ def can_assign_src_to_sink(src: SinkType, sink: Optional[SinkType], strict: bool
9090
return False
9191
if src["type"] == "array" and sink["type"] == "array":
9292
return can_assign_src_to_sink(
93-
cast(MutableSequence[CWLOutputAtomType], src["items"]),
94-
cast(MutableSequence[CWLOutputAtomType], sink["items"]),
93+
cast(MutableSequence[CWLOutputType], src["items"]),
94+
cast(MutableSequence[CWLOutputType], sink["items"]),
9595
strict,
9696
)
9797
if src["type"] == "record" and sink["type"] == "record":

‎cwltool/docker.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,13 @@ def get_from_requirements(
207207
raise WorkflowException("Docker image %s not found" % r["dockerImageId"])
208208

209209
@staticmethod
210-
def append_volume(runtime: List[str], source: str, target: str, writable: bool = False) -> None:
210+
def append_volume(
211+
runtime: List[str],
212+
source: str,
213+
target: str,
214+
writable: bool = False,
215+
skip_mkdirs: bool = False,
216+
) -> None:
211217
"""Add binding arguments to the runtime list."""
212218
options = [
213219
"type=bind",
@@ -221,7 +227,7 @@ def append_volume(runtime: List[str], source: str, target: str, writable: bool =
221227
mount_arg = output.getvalue().strip()
222228
runtime.append(f"--mount={mount_arg}")
223229
# Unlike "--volume", "--mount" will fail if the volume doesn't already exist.
224-
if not os.path.exists(source):
230+
if (not skip_mkdirs) and (not os.path.exists(source)):
225231
os.makedirs(source)
226232

227233
def add_file_or_directory_volume(

‎cwltool/job.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,11 @@
6666
from .cwlprov.provenance_profile import (
6767
ProvenanceProfile, # pylint: disable=unused-import
6868
)
69+
70+
CollectOutputsType = Union[
71+
Callable[[str, int], CWLObjectType], functools.partial[CWLObjectType]
72+
]
73+
6974
needs_shell_quoting_re = re.compile(r"""(^$|[\s|&;()<>\'"$@])""")
7075

7176
FORCE_SHELLED_POPEN = os.getenv("CWLTOOL_FORCE_SHELL_POPEN", "0") == "1"
@@ -112,9 +117,6 @@ def neverquote(string: str, pos: int = 0, endpos: int = 0) -> Optional[Match[str
112117
return None
113118

114119

115-
CollectOutputsType = Union[Callable[[str, int], CWLObjectType], functools.partial]
116-
117-
118120
class JobBase(HasReqsHints, metaclass=ABCMeta):
119121
def __init__(
120122
self,
@@ -144,7 +146,7 @@ def __init__(
144146
self.generatemapper: Optional[PathMapper] = None
145147

146148
# set in CommandLineTool.job(i)
147-
self.collect_outputs = cast(CollectOutputsType, None)
149+
self.collect_outputs = cast("CollectOutputsType", None)
148150
self.output_callback: Optional[OutputCallbackType] = None
149151
self.outdir = ""
150152
self.tmpdir = ""

‎cwltool/loghandler.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
def configure_logging(
1313
stderr_handler: logging.Handler,
14+
no_warnings: bool,
1415
quiet: bool,
1516
debug: bool,
1617
enable_color: bool,
@@ -25,6 +26,8 @@ def configure_logging(
2526
deps_logger.addHandler(stderr_handler)
2627
ss_logger = logging.getLogger("salad")
2728
ss_logger.addHandler(stderr_handler)
29+
if no_warnings:
30+
stderr_handler.setLevel(logging.ERROR)
2831
if quiet:
2932
# Silence STDERR, not an eventual provenance log file
3033
stderr_handler.setLevel(logging.WARN)

‎cwltool/main.py

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@
3434
import argcomplete
3535
import coloredlogs
3636
import ruamel.yaml
37-
from importlib_resources import files
3837
from ruamel.yaml.comments import CommentedMap, CommentedSeq
3938
from ruamel.yaml.main import YAML
4039
from schema_salad.exceptions import ValidationException
@@ -105,10 +104,10 @@
105104
from .utils import (
106105
DEFAULT_TMP_PREFIX,
107106
CWLObjectType,
108-
CWLOutputAtomType,
109107
CWLOutputType,
110108
HasReqsHints,
111109
adjustDirObjs,
110+
files,
112111
normalizeFilesDirs,
113112
processes_to_kill,
114113
trim_listing,
@@ -289,7 +288,7 @@ def realize_input_schema(
289288
_, input_type_name = entry["type"].split("#")
290289
if input_type_name in schema_defs:
291290
entry["type"] = cast(
292-
CWLOutputAtomType,
291+
CWLOutputType,
293292
realize_input_schema(
294293
cast(
295294
MutableSequence[Union[str, CWLObjectType]],
@@ -300,29 +299,29 @@ def realize_input_schema(
300299
)
301300
if isinstance(entry["type"], MutableSequence):
302301
entry["type"] = cast(
303-
CWLOutputAtomType,
302+
CWLOutputType,
304303
realize_input_schema(
305304
cast(MutableSequence[Union[str, CWLObjectType]], entry["type"]),
306305
schema_defs,
307306
),
308307
)
309308
if isinstance(entry["type"], Mapping):
310309
entry["type"] = cast(
311-
CWLOutputAtomType,
310+
CWLOutputType,
312311
realize_input_schema([cast(CWLObjectType, entry["type"])], schema_defs),
313312
)
314313
if entry["type"] == "array":
315314
items = entry["items"] if not isinstance(entry["items"], str) else [entry["items"]]
316315
entry["items"] = cast(
317-
CWLOutputAtomType,
316+
CWLOutputType,
318317
realize_input_schema(
319318
cast(MutableSequence[Union[str, CWLObjectType]], items),
320319
schema_defs,
321320
),
322321
)
323322
if entry["type"] == "record":
324323
entry["fields"] = cast(
325-
CWLOutputAtomType,
324+
CWLOutputType,
326325
realize_input_schema(
327326
cast(MutableSequence[Union[str, CWLObjectType]], entry["fields"]),
328327
schema_defs,
@@ -612,7 +611,7 @@ def loadref(base: str, uri: str) -> Union[CommentedMap, CommentedSeq, str, None]
612611
nestdirs=nestdirs,
613612
)
614613
if sfs is not None:
615-
deps["secondaryFiles"] = cast(MutableSequence[CWLOutputAtomType], mergedirs(sfs))
614+
deps["secondaryFiles"] = cast(MutableSequence[CWLOutputType], mergedirs(sfs))
616615

617616
return deps
618617

@@ -1014,6 +1013,7 @@ def main(
10141013

10151014
configure_logging(
10161015
stderr_handler,
1016+
args.no_warnings,
10171017
args.quiet,
10181018
runtimeContext.debug,
10191019
args.enable_color,

0 commit comments

Comments
 (0)