Document functional interface. (#423)

Author: tobiasraabe

.pre-commit-config.yaml

@@ -69,8 +69,9 @@ repos:
         additional_dependencies: [
             attrs>=21.3.0,
             click,
+            optree,
             pluggy,
-            types-setuptools
+            types-setuptools,
         ]
         pass_filenames: false
 -   repo: https://github.com/executablebooks/mdformat
@@ -113,15 +114,25 @@ repos:
                 docs/source/tutorials/set_up_a_project.md|
                 docs/source/tutorials/write_a_task.md
             )$
-- repo: https://github.com/kynan/nbstripout
-  rev: 0.6.1
-  hooks:
-    - id: nbstripout
+-   repo: https://github.com/nbQA-dev/nbQA
+    rev: 1.7.0
+    hooks:
+    -   id: nbqa-black
+    -   id: nbqa-isort
+    -   id: nbqa-mypy
+        args: [--ignore-missing-imports]
+    -   id: nbqa-ruff
+-   repo: https://github.com/kynan/nbstripout
+    rev: 0.6.1
+    hooks:
+    -   id: nbstripout
+        exclude: (docs)
 -   repo: https://github.com/codespell-project/codespell
     rev: v2.2.5
     hooks:
     -   id: codespell
         additional_dependencies: [tomli]
+        exclude: (\.ipynb)
 -   repo: https://github.com/mgedmin/check-manifest
     rev: "0.49"
     hooks:

docs/source/changes.md

@@ -40,7 +40,11 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
   decorators.
 - {pull}`421` removes the deprecation warning when `produces` is used as an magic
   function keyword to define products.
+- {pull}`423` adds a notebook to explain the functional interface.
 - {pull}`424` fixes problems with {func}`~_pytask.path.import_path`.
+- {pull}`426` publishes the {mod}`pytask.tree_util` module.
+- {pull}`427` fixes type annotations for {attr}`pytask.PTask.depends_on` and
+  {attr}`pytask.PTask.produces`.
 
 ## 0.3.2 - 2023-06-07
 

docs/source/conf.py

@@ -41,6 +41,7 @@
     "sphinx.ext.viewcode",
     "sphinx_copybutton",
     "sphinx_click",
+    "nbsphinx",
     "myst_parser",
     "sphinx_design",
 ]

docs/source/how_to_guides/functional_interface.ipynb

@@ -12,7 +12,7 @@ specific tasks with pytask.
 maxdepth: 1
 ---
 migrating_from_scripts_to_pytask
-invoking_pytask_extended
+functional_interface
 capture_warnings
 how_to_influence_build_order
 hashing_inputs_of_tasks

docs/source/how_to_guides/index.md

@@ -82,3 +82,8 @@ You can do a dry run to see which tasks would be executed without executing them
 
 ```{include} ../_static/md/dry-run.md
 ```
+
+## Functional interface
+
+pytask also has a functional interface that is explained in this
+[article](../how_to_guides/functional_interface.ipynb).

docs/source/how_to_guides/invoking_pytask_extended.md

@@ -67,8 +67,10 @@ ignore = [
 "src/_pytask/dag.py" = ["B023"]
 "tests/test_capture.py" = ["T201", "PT011"]
 "tests/*" = ["D", "ANN", "PLR2004", "S101"]
+"tests/test_jupyter/*" = ["INP001"]
 "scripts/*" = ["D", "INP001"]
 "docs/source/conf.py" = ["D401", "INP001"]
+"docs/source/how_to_guides/functional_interface*" = ["D", "INP", "ARG005"]
 "docs_src/*" = ["ARG001", "D", "INP001"]
 "docs_src/*/*_py310.py" = ["FA102"]
 "docs_src/*/*_py38.py" = ["FA100"]

docs/source/tutorials/invoking_pytask.md

@@ -200,7 +200,9 @@ def _print_collected_tasks(
 
             if show_nodes:
                 nodes = list(tree_leaves(task.depends_on))
-                sorted_nodes = sorted(nodes, key=lambda x: x.name)
+                sorted_nodes = sorted(
+                    nodes, key=lambda x: x.name  # type: ignore[attr-defined]
+                )
                 for node in sorted_nodes:
                     if isinstance(node, PPathNode):
                         if node.path.as_posix() in node.name:
@@ -214,12 +216,15 @@ def _print_collected_tasks(
                         )
                         text = Text(reduced_node_name, style=url_style)
                     else:
-                        text = node.name
+                        text = node.name  # type: ignore[attr-defined]
 
                     task_branch.add(Text.assemble(FILE_ICON, "<Dependency ", text, ">"))
 
                 for node in sorted(
-                    tree_leaves(task.produces), key=lambda x: getattr(x, "path", x.name)
+                    tree_leaves(task.produces),
+                    key=lambda x: getattr(
+                        x, "path", x.name  # type: ignore[attr-defined]
+                    ),
                 ):
                     if isinstance(node, PPathNode):
                         reduced_node_name = str(relative_to(node.path, common_ancestor))
@@ -228,7 +233,7 @@ def _print_collected_tasks(
                         )
                         text = Text(reduced_node_name, style=url_style)
                     else:
-                        text = Text(node.name)
+                        text = Text(node.name)  # type: ignore[attr-defined]
                     task_branch.add(Text.assemble(FILE_ICON, "<Product ", text, ">"))
 
     console.print(tree)

pyproject.toml

@@ -9,6 +9,7 @@
 from typing import Any
 from typing import Callable
 from typing import Iterable
+from typing import Literal
 from typing import TYPE_CHECKING
 
 import rich
@@ -56,7 +57,7 @@
     _IS_LEGACY_WINDOWS = False
 
 
-_COLOR_SYSTEM = None if _IS_LEGACY_WINDOWS else "auto"
+_COLOR_SYSTEM: Literal["auto"] | None = None if _IS_LEGACY_WINDOWS else "auto"
 
 
 _HORIZONTAL_PADDING = (0, 1, 0, 1)

src/_pytask/collect_command.py

@@ -175,7 +175,7 @@ def pytask_execute_task(session: Session, task: PTask) -> bool:
         nodes = tree_leaves(task.produces["return"])
         values = structure_return.flatten_up_to(out)
         for node, value in zip(nodes, values):
-            node.save(value)
+            node.save(value)  # type: ignore[attr-defined]
 
     return True
 

src/_pytask/console.py

@@ -3,7 +3,6 @@
 
 import functools
 from pathlib import Path
-from typing import Any
 
 import optree
 from optree import PyTree
@@ -40,10 +39,3 @@
 tree_flatten_with_path = functools.partial(
     _optree_tree_flatten_with_path, none_is_leaf=True, namespace="pytask"
 )
-
-
-def tree_index(path: tuple[Any, ...], tree: PyTree) -> Any:
-    """Index a tree with a path."""
-    if not path:
-        return tree
-    return tree_index(path[1:], tree[path[0]])

src/_pytask/execute.py

@@ -9,12 +9,10 @@
    "source": [
     "from pathlib import Path\n",
     "\n",
-    "import pytask\n",
+    "from typing_extensions import Annotated\n",
     "\n",
-    "from pytask import ExitCode\n",
-    "from pytask import PathNode\n",
-    "from pytask import PythonNode\n",
-    "from typing_extensions import Annotated"
+    "import pytask\n",
+    "from pytask import ExitCode, PathNode, PythonNode"
    ]
   },
   {
@@ -26,11 +24,14 @@
    "source": [
     "node_text = PythonNode(name=\"text\", hash=True)\n",
     "\n",
+    "\n",
     "def create_text() -> Annotated[int, node_text]:\n",
     "    return \"This is the text.\"\n",
     "\n",
+    "\n",
     "node_file = PathNode.from_path(Path(\"file.txt\").resolve())\n",
     "\n",
+    "\n",
     "def create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]:\n",
     "    return text"
    ]

src/_pytask/tree_util.py

@@ -9,12 +9,10 @@
    "source": [
     "from pathlib import Path\n",
     "\n",
-    "import pytask\n",
+    "from typing_extensions import Annotated\n",
     "\n",
-    "from pytask import ExitCode\n",
-    "from pytask import PathNode\n",
-    "from pytask import PythonNode\n",
-    "from typing_extensions import Annotated"
+    "import pytask\n",
+    "from pytask import ExitCode, PathNode, PythonNode"
    ]
   },
   {
@@ -26,11 +24,14 @@
    "source": [
     "node_text = PythonNode(name=\"text\", hash=True)\n",
     "\n",
+    "\n",
     "def create_text() -> Annotated[int, node_text]:\n",
     "    return \"This is the text.\"\n",
     "\n",
+    "\n",
     "node_file = PathNode(name=\"product\", path=Path(\"file.txt\"))\n",
     "\n",
+    "\n",
     "def create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]:\n",
     "    return text"
    ]