improve test running on mac-os

Author: mr-c

cwltool/argparser.py

@@ -5,7 +5,6 @@
 import urllib
 from typing import (
     Any,
-    AnyStr,
     Callable,
     Dict,
     List,
@@ -18,8 +17,6 @@
     cast,
 )
 
-from schema_salad.ref_resolver import file_uri
-
 from .loghandler import _logger
 from .process import Process, shortname
 from .resolver import ga4gh_tool_registries

cwltool/command_line_tool.py

@@ -15,7 +15,6 @@
 from functools import cmp_to_key, partial
 from typing import (
     Any,
-    Callable,
     Dict,
     Generator,
     List,

cwltool/cuda.py

@@ -1,6 +1,6 @@
 import subprocess  # nosec
 import xml.dom.minidom  # nosec
-from typing import Tuple, cast
+from typing import Tuple
 
 from .loghandler import _logger
 from .utils import CWLObjectType

cwltool/docker.py

@@ -16,7 +16,6 @@
 
 from .builder import Builder
 from .context import RuntimeContext
-from .cuda import cuda_check
 from .docker_id import docker_vm_id
 from .errors import WorkflowException
 from .job import ContainerCommandLineJob

cwltool/provenance_profile.py

@@ -20,15 +20,7 @@
 )
 
 from prov.identifier import Identifier, QualifiedName
-from prov.model import (
-    PROV,
-    PROV_LABEL,
-    PROV_TYPE,
-    PROV_VALUE,
-    ProvDocument,
-    ProvEntity,
-    ProvRecord,
-)
+from prov.model import PROV, PROV_LABEL, PROV_TYPE, PROV_VALUE, ProvDocument, ProvEntity
 from schema_salad.sourceline import SourceLine
 from typing_extensions import TYPE_CHECKING
 

tests/checker_wf/wc.cwl

@@ -1,6 +1,9 @@
 #!/usr/bin/env cwl-runner
 cwlVersion: v1.1
 class: CommandLineTool
+hints:
+  DockerRequirement:
+   dockerPull: docker.io/debian:stable-slim
 inputs:
   intxt:
     type: File

tests/subgraph/count-lines17-wf.cwl.json

@@ -113,6 +113,11 @@
                       "-l"
                     ],
                     "class": "CommandLineTool",
+                    "hints": {
+                      "DockerRequirement": {
+                        "dockerPull": "docker.io/debian:stable-slim"
+                      }
+                    },
                     "id": "count-lines17-wf.cwl@step_step1@run@step_stepX@run@step_stepY@run",
                     "inputs": [
                       {

tests/subgraph/wc-tool.cwl

@@ -3,6 +3,10 @@
 class: CommandLineTool
 cwlVersion: v1.0
 
+hints:
+  DockerRequirement:
+   dockerPull: docker.io/debian:stable-slim
+
 inputs:
   file1: File
 

tests/test_cuda.py

@@ -11,10 +11,10 @@
 from cwltool.job import CommandLineJob
 from cwltool.load_tool import load_tool
 from cwltool.main import main
-from cwltool.pathmapper import MapperEnt, PathMapper
-from cwltool.process import use_custom_schema, use_standard_schema
+from cwltool.pathmapper import PathMapper
+from cwltool.process import use_custom_schema
 from cwltool.stdfsaccess import StdFsAccess
-from cwltool.update import INTERNAL_VERSION, ORIGINAL_CWLVERSION
+from cwltool.update import INTERNAL_VERSION
 from cwltool.utils import CWLObjectType
 
 from .util import get_data, needs_docker, needs_singularity_3_or_newer

tests/test_examples.py

@@ -910,6 +910,7 @@ def test_static_checker() -> None:
         factory.make(get_data("tests/checker_wf/broken-wf3.cwl"))
 
 
+@needs_docker
 def test_circular_dependency_checker() -> None:
     # check that the circular dependency checker raises exception when there is
     # circular dependency in the workflow.
@@ -1119,8 +1120,9 @@ def test_cid_file_dir(tmp_path: Path, factor: str) -> None:
         stderr = re.sub(r"\s\s+", " ", stderr)
         assert "completed success" in stderr
         assert error_code == 0
-        cidfiles_count = sum(1 for _ in tmp_path.glob("**/*"))
-        assert cidfiles_count == 2
+        cidfiles = list(tmp_path.glob("**/*.cid"))
+        cidfiles_count = len(cidfiles)
+        assert cidfiles_count == 2, f"Should be 2 cidfiles, but got {cidfiles}"
 
 
 @needs_docker
@@ -1179,7 +1181,8 @@ def test_cid_file_w_prefix(tmp_path: Path, factor: str) -> None:
             error_code, stdout, stderr = get_main_output(commands)
         finally:
             listing = tmp_path.iterdir()
-            cidfiles_count = sum(1 for _ in tmp_path.glob("**/pytestcid*"))
+            cidfiles = tmp_path.glob("**/pytestcid*")
+            cidfiles_count = sum(1 for _ in cidfiles)
     stderr = re.sub(r"\s\s+", " ", stderr)
     assert "completed success" in stderr
     assert error_code == 0
@@ -1449,6 +1452,7 @@ def test_no_compute_chcksum(tmp_path: Path, factor: str) -> None:
     assert "checksum" not in stdout
 
 
+@needs_docker
 @pytest.mark.parametrize("factor", test_factors)
 def test_bad_userspace_runtime(factor: str) -> None:
     test_file = "tests/wf/wc-tool.cwl"

tests/test_misc_cli.py

@@ -2,7 +2,7 @@
 
 from cwltool.utils import versionstring
 
-from .util import get_data, get_main_output
+from .util import get_data, get_main_output, needs_docker
 
 
 def test_version() -> None:
@@ -44,6 +44,7 @@ def test_basic_pack() -> None:
     assert "$graph" in stdout
 
 
+@needs_docker
 def test_basic_print_subgraph() -> None:
     """Basic test of --print-subgraph. See test_subgraph.py for detailed testing."""
     return_code, stdout, stderr = get_main_output(

tests/test_parallel.py

@@ -5,9 +5,10 @@
 from cwltool.executors import MultithreadedJobExecutor
 from cwltool.factory import Factory
 
-from .util import get_data
+from .util import get_data, needs_docker
 
 
+@needs_docker
 def test_sequential_workflow(tmp_path: Path) -> None:
     test_file = "tests/wf/count-lines1-wf.cwl"
     executor = MultithreadedJobExecutor()
@@ -20,6 +21,7 @@ def test_sequential_workflow(tmp_path: Path) -> None:
     assert echo(file1=file_contents) == {"count_output": 16}
 
 
+@needs_docker
 def test_scattered_workflow() -> None:
     test_file = "tests/wf/scatter-wf4.cwl"
     job_file = "tests/wf/scatter-job2.json"

tests/test_stdout_stderr_log_dir.py

@@ -1,10 +1,7 @@
 import json
-import os
 from pathlib import Path
 
-from cwltool.main import main
-
-from .util import get_data, get_main_output, needs_docker
+from .util import get_data, get_main_output
 
 
 def test_log_dir_echo_output(tmp_path: Path) -> None:

tests/test_subgraph.py

@@ -240,6 +240,7 @@ def test_single_process_packed_subwf_step(tmp_path: Path) -> None:
     )
 
 
+@needs_docker
 def test_single_process_subwf_subwf_inline_step() -> None:
     """Test --single-process on an inline sub-sub-workflow step."""
     err_code, stdout, stderr = get_main_output(
@@ -350,6 +351,7 @@ def test_print_targets_embedded_reqsinherit() -> None:
     assert err_code == 0
 
 
+@needs_docker
 def test_print_targets_embedded_sub_subwfs() -> None:
     """Confirm that --print-targets works with inline sub-sub-workflows."""
     err_code, stdout, stderr = get_main_output(

tests/test_toolargparse.py

@@ -1,7 +1,7 @@
 import argparse
 from io import StringIO
 from pathlib import Path
-from typing import Any, Callable, Dict, List
+from typing import Callable, List
 
 import pytest
 

tests/test_udocker.py

@@ -50,8 +50,8 @@ def test_udocker_usage_should_not_write_cid_file(udocker: str, tmp_path: Path) -
     """Confirm that no cidfile is made when udocker is used."""
 
     with working_directory(tmp_path):
-        test_file = "tests/wf/wc-tool.cwl"
-        job_file = "tests/wf/wc-job.json"
+        test_file = "tests/echo-position-expr.cwl"
+        job_file = "tests/echo-position-echo-position-expr-job.yml"
         error_code, stdout, stderr = get_main_output(
             [
                 "--debug",

tests/wf/iwdr_permutations.cwl

@@ -74,13 +74,13 @@ stdout: log.txt
 baseCommand: [bash, -c]
 arguments:
   - |
-    find . | sort
+    find . | grep -v '\.docker' | sort
     find /my_path | sort
     find /my_other_path | sort
     echo "a" > first_writable_file
     echo "b" > /my_path/third_writable_file
     touch fifth_writable_directory/c
     touch /my_path/seventh_writable_directory/d
-    find . | sort
+    find . | grep -v '\.docker' | sort
     find /my_path | sort
     find /my_other_path | sort

tests/wf/iwdr_permutations_nocontainer.cwl

@@ -38,6 +38,6 @@ outputs:
 baseCommand: [bash, -c]
 arguments:
   - |
-    find .
+    find . | grep -v '\.docker' | sort
     echo "a" > first_writable_file
     touch fifth_writable_directory/c

tests/wf/wc-tool.cwl

@@ -3,6 +3,10 @@
 class: CommandLineTool
 cwlVersion: v1.0
 
+hints:
+  DockerRequirement:
+   dockerPull: docker.io/debian:stable-slim
+
 inputs:
   file1: File