singularity: support dockerFile (#1938)

* allow user to set temp dir with APPTAINER_TMPDIR
* singularity build: automatically try fakeroot mode if proot is missing

---------

Co-authored-by: Brandon Walker <[email protected]>
Co-authored-by: Michael R. Crusoe <[email protected]>

Author: 3 people

.github/workflows/ci-tests.yml

@@ -77,7 +77,7 @@ jobs:
           key: mypy-${{ env.py-semver }}
 
       - name: Test with tox
-        run: tox
+        run: APPTAINER_TMPDIR=${RUNNER_TEMP} tox
 
       - name: Upload coverage to Codecov
         if: ${{ matrix.step == 'unit' }}
@@ -156,7 +156,7 @@ jobs:
           chmod a-w .
 
       - name: run tests
-        run: make test
+        run: APPTAINER_TMPDIR=${RUNNER_TEMP} make test
 
 
   conformance_tests:

cwltool/singularity.py

@@ -10,6 +10,9 @@
 from typing import Callable, Dict, List, MutableMapping, Optional, Tuple, cast
 
 from schema_salad.sourceline import SourceLine
+from spython.main import Client
+from spython.main.parse.parsers.docker import DockerParser
+from spython.main.parse.writers.singularity import SingularityWriter
 
 from .builder import Builder
 from .context import RuntimeContext
@@ -140,6 +143,7 @@ def __init__(
     def get_image(
         dockerRequirement: Dict[str, str],
         pull_image: bool,
+        tmp_outdir_prefix: str,
         force_pull: bool = False,
     ) -> bool:
         """
@@ -162,7 +166,35 @@ def get_image(
         elif is_version_2_6() and "SINGULARITY_PULLFOLDER" in os.environ:
             cache_folder = os.environ["SINGULARITY_PULLFOLDER"]
 
-        if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
+        if "dockerFile" in dockerRequirement:
+            if cache_folder is None:  # if environment variables were not set
+                cache_folder = create_tmp_dir(tmp_outdir_prefix)
+
+            absolute_path = os.path.abspath(cache_folder)
+            dockerfile_path = os.path.join(absolute_path, "Dockerfile")
+            singularityfile_path = dockerfile_path + ".def"
+            # if you do not set APPTAINER_TMPDIR will crash
+            # WARNING: 'nodev' mount option set on /tmp, it could be a
+            #          source of failure during build process
+            # FATAL:   Unable to create build: 'noexec' mount option set on
+            #          /tmp, temporary root filesystem won't be usable at this location
+            with open(dockerfile_path, "w") as dfile:
+                dfile.write(dockerRequirement["dockerFile"])
+
+            singularityfile = SingularityWriter(DockerParser(dockerfile_path).parse()).convert()
+            with open(singularityfile_path, "w") as file:
+                file.write(singularityfile)
+
+            os.environ["APPTAINER_TMPDIR"] = absolute_path
+            singularity_options = ["--fakeroot"] if not shutil.which("proot") else []
+            Client.build(
+                recipe=singularityfile_path,
+                build_folder=absolute_path,
+                sudo=False,
+                options=singularity_options,
+            )
+            found = True
+        elif "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
             match = re.search(pattern=r"([a-z]*://)", string=dockerRequirement["dockerPull"])
             img_name = _normalize_image_id(dockerRequirement["dockerPull"])
             candidates.append(img_name)
@@ -243,13 +275,6 @@ def get_image(
                     check_call(cmd, stdout=sys.stderr)  # nosec
                     found = True
 
-            elif "dockerFile" in dockerRequirement:
-                raise SourceLine(
-                    dockerRequirement, "dockerFile", WorkflowException, debug
-                ).makeError(
-                    "dockerFile is not currently supported when using the "
-                    "Singularity runtime for Docker containers."
-                )
             elif "dockerLoad" in dockerRequirement:
                 if is_version_3_1_or_newer():
                     if "dockerImageId" in dockerRequirement:
@@ -298,7 +323,7 @@ def get_from_requirements(
         if not bool(shutil.which("singularity")):
             raise WorkflowException("singularity executable is not available")
 
-        if not self.get_image(cast(Dict[str, str], r), pull_image, force_pull):
+        if not self.get_image(cast(Dict[str, str], r), pull_image, tmp_outdir_prefix, force_pull):
             raise WorkflowException("Container image {} not found".format(r["dockerImageId"]))
 
         return os.path.abspath(cast(str, r["dockerImageId"]))

mypy-stubs/spython/main/__init__.pyi

@@ -0,0 +1,9 @@
+from typing import Iterator, Optional
+
+from .base import Client as _BaseClient
+from .build import build as base_build
+
+class _Client(_BaseClient):
+    build = base_build
+
+Client = _Client()

mypy-stubs/spython/main/base/__init__.pyi

@@ -0,0 +1,3 @@
+class Client:
+    def __init__(self) -> None: ...
+    def version(self) -> str: ...

mypy-stubs/spython/main/build.pyi

@@ -0,0 +1,23 @@
+from typing import Iterator, Optional
+
+from .base import Client
+
+def build(
+    self: Client,
+    recipe: Optional[str] = ...,
+    image: Optional[str] = ...,
+    isolated: Optional[bool] = ...,
+    sandbox: Optional[bool] = ...,
+    writable: Optional[bool] = ...,
+    build_folder: Optional[str] = ...,
+    robot_name: Optional[bool] = ...,
+    ext: Optional[str] = ...,
+    sudo: Optional[bool] = ...,
+    stream: Optional[bool] = ...,
+    force: Optional[bool] = ...,
+    options: Optional[list[str]] | None = ...,
+    quiet: Optional[bool] = ...,
+    return_result: Optional[bool] = ...,
+    sudo_options: Optional[str | list[str]] = ...,
+    singularity_options: Optional[list[str]] = ...,
+) -> tuple[str, Iterator[str]]: ...

mypy-stubs/spython/main/parse/parsers/base.pyi

@@ -0,0 +1,14 @@
+import abc
+
+from ..recipe import Recipe
+
+class ParserBase(metaclass=abc.ABCMeta):
+    filename: str
+    lines: list[str]
+    args: dict[str, str]
+    active_layer: str
+    active_layer_num: int
+    recipe: dict[str, Recipe]
+    def __init__(self, filename: str, load: bool = ...) -> None: ...
+    @abc.abstractmethod
+    def parse(self) -> dict[str, Recipe]: ...

mypy-stubs/spython/main/parse/parsers/docker.pyi

@@ -0,0 +1,7 @@
+from ..recipe import Recipe
+from .base import ParserBase as ParserBase
+
+class DockerParser(ParserBase):
+    name: str
+    def __init__(self, filename: str = ..., load: bool = ...) -> None: ...
+    def parse(self) -> dict[str, Recipe]: ...

mypy-stubs/spython/main/parse/recipe.pyi

@@ -0,0 +1,19 @@
+from typing import Optional
+
+class Recipe:
+    cmd: Optional[str]
+    comments: list[str]
+    entrypoint: Optional[str]
+    environ: list[str]
+    files: list[str]
+    layer_files: dict[str, str]
+    install: list[str]
+    labels: list[str]
+    ports: list[str]
+    test: Optional[str]
+    volumes: list[str]
+    workdir: Optional[str]
+    layer: int
+    fromHeader: Optional[str]
+    source: Optional[Recipe]
+    def __init__(self, recipe: Optional[Recipe] = ..., layer: int = ...) -> None: ...

mypy-stubs/spython/main/parse/writers/base.pyi

@@ -0,0 +1,6 @@
+from ..recipe import Recipe
+
+class WriterBase:
+    recipe: dict[str, Recipe]
+    def __init__(self, recipe: dict[str, Recipe] | None = ...) -> None: ...
+    def write(self, output_file: str | None = ..., force: bool = ...) -> None: ...

mypy-stubs/spython/main/parse/writers/singularity.pyi

@@ -0,0 +1,10 @@
+from typing import Optional
+
+from ..recipe import Recipe
+from .base import WriterBase as WriterBase
+
+class SingularityWriter(WriterBase):
+    name: str
+    def __init__(self, recipe: Optional[dict[str, Recipe]] = ...) -> None: ...
+    def validate(self) -> None: ...
+    def convert(self, runscript: str = ..., force: bool = ...) -> str: ...