Merge branch 'main' into ci_apptainer

Author: mr-c

.flake8

@@ -0,0 +1,8 @@
+[flake8]
+ignore = E203,W503
+max-line-length = 100
+select = B,C,E,F,W,T4
+exclude = cwltool/schemas
+extend-ignore = E501,B905
+# when Python 3.10 is the minimum version, re-enable check B905 for zip + strict
+extend-select = B9

.git-blame-ignore-revs

@@ -0,0 +1,2 @@
+# black + isort
+69e709fde4a3afada10d441cb9e0a85f8a95a3d4

.github/workflows/ci-tests.yml

@@ -57,7 +57,7 @@ jobs:
       - name: Upgrade setuptools and install tox
         run: |
           pip install -U pip setuptools wheel
-          pip install tox tox-gh-actions
+          pip install "tox<4" "tox-gh-actions<3"
 
       - name: MyPy cache
         if: ${{ matrix.step == 'mypy' }}
@@ -86,8 +86,8 @@ jobs:
         step: [lint-readme, pydocstyle]
 
     env:
-      py-semver: 3.11
-      TOXENV: ${{ format('py310-{0}', matrix.step) }}
+      py-semver: "3.11"
+      TOXENV: ${{ format('py311-{0}', matrix.step) }}
 
     steps:
       - uses: actions/checkout@v3
@@ -98,17 +98,12 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ env.py-semver }}
-
-      - name: Cache for pip
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ matrix.step }}-${{ hashFiles('requirements.txt') }}
+          cache: pip
 
       - name: Upgrade setuptools and install tox
         run: |
           pip install -U pip setuptools wheel
-          pip install tox tox-gh-actions
+          pip install "tox<4" "tox-gh-actions<3"
 
       - if: ${{ matrix.step == 'pydocstyle' && github.event_name == 'pull_request'}}
         name: Create local branch for diff-quality for PRs
@@ -119,7 +114,6 @@ jobs:
 
   release_test:
     name: cwl-utils release test
-
     runs-on: ubuntu-22.04
 
     steps:
@@ -134,12 +128,10 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.11
-
-      - name: Cache for pip
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-release-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
+          cache: pip
+          cache-dependency-path: |
+            requirements.txt
+            test-requirements.txt
 
       - name: Install packages
         run: |

.readthedocs.yml

@@ -12,9 +12,14 @@ sphinx:
 # Optionally build your docs in additional formats such as PDF and ePub
 formats: all
 
-# Optionally set the version of Python and requirements required to build your docs
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+  apt_packages:
+    - graphviz
+
 python:
-  version: 3.7
   install:
    - requirements: docs/requirements.txt
    - method: pip

CONTRIBUTING.md

@@ -3,6 +3,9 @@ Style guide:
 - Python 3.6+ compatible code
 - PEP-484 type hints
 
+It is suggested that you run `git config blame.ignoreRevsFile .git-blame-ignore-revs`
+to filter out mass-formatting commits from `git blame`.
+
 In order to contribute to the development of ``cwl-utils``, you need to install cwl-utils from source (preferably in a virtual environment):
 Here's a rough guide (improvements are welcome!) 
 - Install virtualenv via pip: ``pip install virtualenv``

MANIFEST.in

@@ -13,6 +13,9 @@ include testdata/*.yaml
 include testdata/*.input
 include testdata/*.ttl
 include testdata/*.owl
+include testdata/checker_wf/*.cwl
 include cwl_utils/py.typed
+include docs/conf.py docs/Makefile docs/_static/favicon.ico docs/requirements.txt
+include docs/*.rst
 global-exclude *~
 global-exclude *.pyc

Makefile

@@ -28,7 +28,7 @@ EXTRAS=
 # `[[` conditional expressions.
 PYSOURCES=$(filter-out $(MODULE)/parser/cwl_v%,$(shell find $(MODULE) -name "*.py")) \
 	  $(wildcard tests/*.py) create_cwl_from_objects.py load_cwl_by_path.py \
-	  setup.py ${MODULE}/parser/cwl_v1_?_utils.py
+	  setup.py ${MODULE}/parser/cwl_v1_?_utils.py docs/conf.py
 DEVPKGS=diff_cover black pylint pep257 pydocstyle flake8 tox tox-pyenv \
 	isort wheel autoflake flake8-bugbear pyupgrade bandit \
 	-rtest-requirements.txt -rmypy-requirements.txt
@@ -109,8 +109,8 @@ codespell:
 	codespell -w $(shell git ls-files | grep -v mypy-stubs)
 
 ## format                 : check/fix all code indentation and formatting (runs black)
-format: $(PYSOURCES)
-	black $^
+format: $(PYSOURCES) FORCE
+	black $(PYSOURCES)
 
 format-check: $(PYSOURCES)
 	black --diff --check $^

cwl_utils/__meta__.py

@@ -1,3 +1,3 @@
 # SPDX-License-Identifier: Apache-2.0
 """Global version number for the cwl_utils package."""
-__version__ = "0.21"
+__version__ = "0.22"

cwl_utils/cite_extract.py

@@ -10,7 +10,7 @@
 
 
 def arg_parser() -> argparse.ArgumentParser:
-    """Argument parser."""
+    """Construct the argument parser."""
     parser = argparse.ArgumentParser(
         description="Print information about software used in a CWL document (Workflow or CommandLineTool). "
         "For CWL Workflows, all steps will also be searched (recursively)."

cwl_utils/cwl_v1_0_expression_refactor.py

@@ -350,7 +350,7 @@ def generate_etool_from_expr(
         elif self_type.type:
             new_type = clean_type_ids(self_type.type)
         else:
-            raise WorkflowException(f"Don't know how to make type from '{self_type}'.")
+            raise WorkflowException(f"Don't know how to make type from {self_type!r}.")
         inputs.append(
             cwl.InputParameter(
                 id="self",
@@ -652,7 +652,8 @@ def process_workflow_reqs_and_hints(
     """
     # TODO: consolidate the generated etools/cltools into a single "_expression_workflow_reqs" step
     # TODO: support resourceReq.* references to Workflow.inputs?
-    #       ^ By refactoring replace_expr_etool to allow multiple inputs, and connecting all workflow inputs to the generated step
+    #       ^ By refactoring replace_expr_etool to allow multiple inputs,
+    #         and connecting all workflow inputs to the generated step
     modified = False
     inputs = empty_inputs(workflow)
     generated_res_reqs: List[Tuple[str, Union[int, str]]] = []
@@ -837,7 +838,8 @@ def process_workflow_reqs_and_hints(
                                                     raise_type=WorkflowException,
                                                 ).makeError(
                                                     f"Entry {index},"
-                                                    + "Invalid CWL, if 'entry' is a string, then entryName must be specified."
+                                                    + "Invalid CWL, if 'entry' "
+                                                    "is a string, then entryName must be specified."
                                                 )
                                             expr = (
                                                 '${return {"class": "File", "basename": "'
@@ -979,7 +981,8 @@ def process_level_reqs(
     # - Replace the CWL Expression inplace with a CWL parameter reference
     # - Don't create a new Requirement, nor delete the existing Requirement
     # - the Process is passed to replace_expr_with_etool for later searching for JS expressionLibs
-    # - in addition to adding the input to the step for the ETool/CTool result, add it to the Process.inputs as well
+    # - in addition to adding the input to the step for the ETool/CTool result,
+    #   add it to the Process.inputs as well
     if not process.requirements:
         return False
     modified = False
@@ -1122,7 +1125,7 @@ def process_level_reqs(
                                         entryname = (
                                             entry.entryname
                                             if entryname_expr
-                                            else f'"{entry.entryname}"'
+                                            else f'"{entry.entryname}"'  # noqa: B907
                                         )
                                         new_expression = (
                                             "${var result; var entryname = "
@@ -1914,15 +1917,15 @@ def workflow_step_to_InputParameters(
                 for p in param:
                     if not p.type:
                         raise WorkflowException(
-                            f"Don't know how to get type id for '{p}'."
+                            f"Don't know how to get type id for {p!r}."
                         )
                     p.id = inp_id
                     p.type = clean_type_ids(p.type)
                     params.append(p)
             else:
                 if not param.type:
                     raise WorkflowException(
-                        f"Don't know how to get type id for '{param}'."
+                        f"Don't know how to get type id for {param!r}."
                     )
                 param.id = inp_id
                 param.type = clean_type_ids(param.type)

cwl_utils/cwl_v1_1_expression_refactor.py

@@ -183,7 +183,7 @@ def etool_to_cltool(
         contents += "\n" + "\n".join(expressionLib)
     contents += (
         """
-    var ret = function(){"""
+var ret = function(){"""
         + escape_expression_field(etool.expression.strip()[2:-1])
         + """}();
 process.stdout.write(JSON.stringify(ret));"""
@@ -389,7 +389,7 @@ def generate_etool_from_expr(
         expression += "\n  var self=inputs.self;"
     expression += (
         """
-      return {"result": function(){"""
+  return {"result": function(){"""
         + expr[2:-2]
         + """}()};
  }"""
@@ -652,7 +652,8 @@ def process_workflow_reqs_and_hints(
     """
     # TODO: consolidate the generated etools/cltools into a single "_expression_workflow_reqs" step
     # TODO: support resourceReq.* references to Workflow.inputs?
-    #       ^ By refactoring replace_expr_etool to allow multiple inputs, and connecting all workflow inputs to the generated step
+    #       ^ By refactoring replace_expr_etool to allow multiple inputs,
+    #         and connecting all workflow inputs to the generated step
     modified = False
     inputs = empty_inputs(workflow)
     generated_res_reqs: List[Tuple[str, Union[int, str]]] = []
@@ -837,7 +838,8 @@ def process_workflow_reqs_and_hints(
                                                     raise_type=WorkflowException,
                                                 ).makeError(
                                                     f"Entry {index},"
-                                                    + "Invalid CWL, if 'entry' is a string, then entryName must be specified."
+                                                    + "Invalid CWL, if 'entry' "
+                                                    "is a string, then entryName must be specified."
                                                 )
                                             expr = (
                                                 '${return {"class": "File", "basename": "'
@@ -979,7 +981,8 @@ def process_level_reqs(
     # - Replace the CWL Expression inplace with a CWL parameter reference
     # - Don't create a new Requirement, nor delete the existing Requirement
     # - the Process is passed to replace_expr_with_etool for later searching for JS expressionLibs
-    # - in addition to adding the input to the step for the ETool/CTool result, add it to the Process.inputs as well
+    # - in addition to adding the input to the step for the ETool/CTool result,
+    #   add it to the Process.inputs as well
     if not process.requirements:
         return False
     modified = False
@@ -1122,7 +1125,7 @@ def process_level_reqs(
                                         entryname = (
                                             entry.entryname
                                             if entryname_expr
-                                            else f'"{entry.entryname}"'
+                                            else f'"{entry.entryname}"'  # noqa: B907
                                         )
                                         new_expression = (
                                             "${var result; var entryname = "
@@ -1716,7 +1719,7 @@ def generate_etool_from_expr2(
         expression += f"\n  var self=inputs.{self_name};"
     expression += (
         """
-      return {"result": function(){"""
+  return {"result": function(){"""
         + expr[2:-2]
         + """}()};
  }"""

cwl_utils/cwl_v1_2_expression_refactor.py

@@ -183,7 +183,7 @@ def etool_to_cltool(
         contents += "\n" + "\n".join(expressionLib)
     contents += (
         """
-    var ret = function(){"""
+var ret = function(){"""
         + escape_expression_field(etool.expression.strip()[2:-1])
         + """}();
 process.stdout.write(JSON.stringify(ret));"""
@@ -389,7 +389,7 @@ def generate_etool_from_expr(
         expression += "\n  var self=inputs.self;"
     expression += (
         """
-      return {"result": function(){"""
+  return {"result": function(){"""
         + expr[2:-2]
         + """}()};
  }"""
@@ -747,7 +747,8 @@ def process_workflow_reqs_and_hints(
     """
     # TODO: consolidate the generated etools/cltools into a single "_expression_workflow_reqs" step
     # TODO: support resourceReq.* references to Workflow.inputs?
-    #       ^ By refactoring replace_expr_etool to allow multiple inputs, and connecting all workflow inputs to the generated step
+    #       ^ By refactoring replace_expr_etool to allow multiple inputs,
+    #         and connecting all workflow inputs to the generated step
     modified = False
     inputs = empty_inputs(workflow)
     generated_res_reqs: List[Tuple[str, Union[int, str]]] = []
@@ -932,7 +933,8 @@ def process_workflow_reqs_and_hints(
                                                     raise_type=WorkflowException,
                                                 ).makeError(
                                                     f"Entry {index},"
-                                                    + "Invalid CWL, if 'entry' is a string, then entryName must be specified."
+                                                    + "Invalid CWL, if 'entry' "
+                                                    "is a string, then entryName must be specified."
                                                 )
                                             expr = (
                                                 '${return {"class": "File", "basename": "'
@@ -1074,7 +1076,8 @@ def process_level_reqs(
     # - Replace the CWL Expression inplace with a CWL parameter reference
     # - Don't create a new Requirement, nor delete the existing Requirement
     # - the Process is passed to replace_expr_with_etool for later searching for JS expressionLibs
-    # - in addition to adding the input to the step for the ETool/CTool result, add it to the Process.inputs as well
+    # - in addition to adding the input to the step for the ETool/CTool result,
+    #   add it to the Process.inputs as well
     if not process.requirements:
         return False
     modified = False
@@ -1217,7 +1220,7 @@ def process_level_reqs(
                                         entryname = (
                                             entry.entryname
                                             if entryname_expr
-                                            else f'"{entry.entryname}"'
+                                            else f'"{entry.entryname}"'  # noqa: B907
                                         )
                                         new_expression = (
                                             "${var result; var entryname = "
@@ -1811,7 +1814,7 @@ def generate_etool_from_expr2(
         expression += f"\n  var self=inputs.{self_name};"
     expression += (
         """
-      return {"result": function(){"""
+  return {"result": function(){"""
         + expr[2:-2]
         + """}()};
  }"""

cwl_utils/errors.py

@@ -3,23 +3,23 @@
 
 
 class ArrayMissingItems(BaseException):
-    """From https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py"""
+    """From https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py ."""
 
 
 class JavascriptException(Exception):
     pass
 
 
 class MissingKeyField(BaseException):
-    """From https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py"""
+    """From https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py ."""
 
 
 class MissingTypeName(BaseException):
-    """From https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py"""
+    """From https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py ."""
 
 
 class RecordMissingFields(BaseException):
-    """From https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py"""
+    """From https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py ."""
 
 
 class SubstitutionError(Exception):