diff --git a/docs/source/_static/md/markers.md b/docs/source/_static/md/markers.md index 2ce8c7d7..99e8c202 100644 --- a/docs/source/_static/md/markers.md +++ b/docs/source/_static/md/markers.md @@ -6,10 +6,6 @@ $ pytask markers ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ ┃ Marker ┃ Description ┃ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ pytask.mark.depends_on │ Add dependencies to a task. See this │ -│ │ tutorial for more information: │ -│ │ https://bit.ly/3JlxylS. │ -│ │ │ │ pytask.mark.persist │ Prevent execution of a task if all │ │ │ products exist and even ifsomething has │ │ │ changed (dependencies, source file, │ @@ -21,10 +17,6 @@ $ pytask markers │ │ another run will skip the task with │ │ │ success. │ │ │ │ -│ pytask.mark.produces │ Add products to a task. See this │ -│ │ tutorial for more information: │ -│ │ https://bit.ly/3JlxylS. │ -│ │ │ │ pytask.mark.skip │ Skip a task and all its dependent tasks.│ │ │ │ │ pytask.mark.skip_ancestor_failed │ Internal decorator applied to tasks if │ diff --git a/docs/source/changes.md b/docs/source/changes.md index ebea8973..2e53e5ac 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -5,10 +5,12 @@ chronological order. Releases follow [semantic versioning](https://semver.org/) releases are available on [PyPI](https://pypi.org/project/pytask) and [Anaconda.org](https://anaconda.org/conda-forge/pytask). -## 0.4.6 +## 0.5.0 - 2024-xx-xx - {pull}`548` fixes the type hints for {meth}`~pytask.Task.execute` and {meth}`~pytask.TaskWithoutPath.execute`. Thanks to {user}`Ostheer`. +- {pull}`551` removes the deprecated `@pytask.mark.depends_on` and + `@pytask.mark.produces`. ## 0.4.5 - 2024-01-09 diff --git a/docs/source/how_to_guides/interfaces_for_dependencies_products.md b/docs/source/how_to_guides/interfaces_for_dependencies_products.md index 774240cb..5a20a4a6 100644 --- a/docs/source/how_to_guides/interfaces_for_dependencies_products.md +++ b/docs/source/how_to_guides/interfaces_for_dependencies_products.md @@ -16,12 +16,12 @@ In general, pytask regards everything as a task dependency if it is not marked a product. Thus, you can also think of the following examples as how to inject values into a task. When we talk about products later, the same interfaces will be used. -| | `def task(arg: ... = ...)` | `Annotated[..., value]` | `@task(kwargs=...)` | `@pytask.mark.depends_on(...)` | -| --------------------------------------- | :------------------------: | :---------------------: | :-----------------: | :----------------------------: | -| Not deprecated | ✅ | ✅ | ✅ | ❌ | -| No type annotations required | ✅ | ❌ | ✅ | ✅ | -| Flexible choice of argument name | ✅ | ✅ | ✅ | ❌ | -| Supports third-party functions as tasks | ❌ | ❌ | ✅ | ❌ | +| | `def task(arg: ... = ...)` | `Annotated[..., value]` | `@task(kwargs=...)` | +| --------------------------------------- | :------------------------: | :---------------------: | :-----------------: | +| Not deprecated | ✅ | ✅ | ✅ | +| No type annotations required | ✅ | ❌ | ✅ | +| Flexible choice of argument name | ✅ | ✅ | ✅ | +| Supports third-party functions as tasks | ❌ | ❌ | ✅ | (default-argument)= @@ -58,13 +58,13 @@ dictionary. It applies to dependencies and products alike. ## Products -| | `def task(arg: Annotated[..., Product] = ...)` | `Annotated[..., value, Product]` | `produces` | `@task(produces=...)` | `def task() -> Annotated[..., value]` | `@pytask.mark.produces(...)` | -| --------------------------------------------------------- | :--------------------------------------------: | :------------------------------: | :--------: | :-------------------: | :-----------------------------------: | :--------------------------: | -| Not deprecated | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | -| No type annotations required | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | -| Flexible choice of argument name | ✅ | ✅ | ❌ | ✅ | ➖ | ❌ | -| Supports third-party functions as tasks | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | -| Allows to pass custom node while preserving type of value | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | +| | `def task(arg: Annotated[..., Product] = ...)` | `Annotated[..., value, Product]` | `produces` | `@task(produces=...)` | `def task() -> Annotated[..., value]` | +| --------------------------------------------------------- | :--------------------------------------------: | :------------------------------: | :--------: | :-------------------: | :-----------------------------------: | +| Not deprecated | ✅ | ✅ | ✅ | ✅ | ✅ | +| No type annotations required | ❌ | ❌ | ✅ | ✅ | ❌ | +| Flexible choice of argument name | ✅ | ✅ | ❌ | ✅ | ➖ | +| Supports third-party functions as tasks | ❌ | ❌ | ❌ | ✅ | ❌ | +| Allows to pass custom node while preserving type of value | ❌ | ✅ | ✅ | ✅ | ✅ | ### `Product` annotation diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md index b7efd089..e7c07527 100644 --- a/docs/source/reference_guides/api.md +++ b/docs/source/reference_guides/api.md @@ -63,36 +63,16 @@ The remaining exceptions convey specific errors. ## Marks -pytask uses marks to attach additional information to task functions which is processed -by the host or by plugins. The following marks are available by default. +pytask uses marks to attach additional information to task functions that the host or +plugins process. The following marks are available by default. ### Built-in marks ```{eval-rst} -.. function:: pytask.mark.depends_on(objects: Any | Iterable[Any] | dict[Any, Any]) - - Specify dependencies for a task. - - :type objects: Any | Iterable[Any] | dict[Any, Any] - :param objects: - Can be any valid Python object or an iterable of any Python objects. To be - valid, it must be parsed by some hook implementation for the - :func:`_pytask.hookspecs.pytask_collect_node` entry-point. - .. function:: pytask.mark.persist() A marker for a task which should be persisted. -.. function:: pytask.mark.produces(objects: Any | Iterable[Any] | dict[Any, Any]) - - Specify products of a task. - - :type objects: Any | Iterable[Any] | dict[Any, Any] - :param objects: - Can be any valid Python object or an iterable of any Python objects. To be - valid, it must be parsed by some hook implementation for the - :func:`_pytask.hookspecs.pytask_collect_node` entry-point. - .. function:: pytask.mark.skipif(condition: bool, *, reason: str) Skip a task based on a condition and provide a necessary reason. @@ -251,10 +231,8 @@ Nodes are the interface for different kinds of dependencies or products. To parse dependencies and products from nodes, use the following functions. ```{eval-rst} -.. autofunction:: pytask.depends_on .. autofunction:: pytask.parse_dependencies_from_task_function .. autofunction:: pytask.parse_products_from_task_function -.. autofunction:: pytask.produces ``` ## Tasks diff --git a/docs/source/tutorials/configuration.md b/docs/source/tutorials/configuration.md index 7492e2b3..20328084 100644 --- a/docs/source/tutorials/configuration.md +++ b/docs/source/tutorials/configuration.md @@ -21,7 +21,7 @@ configuration file. ```toml [tool.pytask.ini_options] -paths = "src" +paths = ["src"] ``` ## The location diff --git a/docs/source/tutorials/defining_dependencies_products.md b/docs/source/tutorials/defining_dependencies_products.md index 0472b666..71831554 100644 --- a/docs/source/tutorials/defining_dependencies_products.md +++ b/docs/source/tutorials/defining_dependencies_products.md @@ -11,11 +11,6 @@ You find a tutorial on type hints {doc}`here <../type_hints>`. If you want to avoid type annotations for now, look at the tab named `produces`. -```{warning} -The `Decorators` tab documents the deprecated approach that should not be used anymore -and will be removed in version v0.5. -``` - ```{seealso} In this tutorial, we only deal with local files. If you want to use pytask with files online, S3, GCP, Azure, etc., read the @@ -89,26 +84,6 @@ passed to this argument is automatically treated as a task product. Here, we pas path as the default argument. ```` - -````{tab-item} Decorators -:sync: decorators - -```{warning} -This approach is deprecated and will be removed in v0.5 -``` - -```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_products_decorators.py -:emphasize-lines: 9, 10 -``` - -The {func}`@pytask.mark.produces ` marker attaches a product to a -task. After the task has finished, pytask will check whether the file exists. - -Add `produces` as an argument of the task function to get access to the same path inside -the task function. - -```` - ````` ```{tip} @@ -170,24 +145,6 @@ pytask assumes that all function arguments that are not passed to the argument :emphasize-lines: 9 ``` -```` - -````{tab-item} Decorators -:sync: decorators - -```{warning} -This approach is deprecated and will be removed in v0.5 -``` - -Equivalent to products, you can use the -{func}`@pytask.mark.depends_on ` decorator to specify that -`data.pkl` is a dependency of the task. Use `depends_on` as a function argument to -access the dependency path inside the function and load the data. - -```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_dependencies_decorators.py -:emphasize-lines: 9, 11 -``` - ```` ````` @@ -228,25 +185,6 @@ are assumed to point to a location relative to the task module. :emphasize-lines: 4 ``` -```` - -````{tab-item} Decorators -:sync: decorators - -```{warning} -This approach is deprecated and will be removed in v0.5 -``` - -You can also use absolute and relative paths as strings that obey the same rules as the -{class}`pathlib.Path`. - -```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_relative_decorators.py -:emphasize-lines: 6 -``` - -If you use `depends_on` or `produces` as arguments for the task function, you will have -access to the paths of the targets as {class}`pathlib.Path`. - ```` ````` @@ -286,7 +224,7 @@ structures if needed. ```` -````{tab-item} prodouces +````{tab-item} produces :sync: produces If your task has multiple products, group them in one container like a dictionary @@ -300,117 +238,6 @@ You can do the same with dependencies. ```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_multiple2_produces.py ``` -```` - -````{tab-item} Decorators -:sync: decorators - -```{warning} -This approach is deprecated and will be removed in v0.5 -``` - -The easiest way to attach multiple dependencies or products to a task is to pass a -{class}`dict` (highly recommended), {class}`list`, or another iterator to the marker -containing the paths. - -To assign labels to dependencies or products, pass a dictionary. For example, - -```python -from typing import Dict - - -@pytask.mark.produces({"first": BLD / "data_0.pkl", "second": BLD / "data_1.pkl"}) -def task_create_random_data(produces: Dict[str, Path]) -> None: - ... -``` - -Then, use `produces` inside the task function. - -```pycon ->>> produces["first"] -BLD / "data_0.pkl" - ->>> produces["second"] -BLD / "data_1.pkl" -``` - -You can also use lists and other iterables. - -```python -@pytask.mark.produces([BLD / "data_0.pkl", BLD / "data_1.pkl"]) -def task_create_random_data(produces): - ... -``` - -Inside the function, the arguments `depends_on` or `produces` become a dictionary where -keys are the positions in the list. - -```pycon ->>> produces -{0: BLD / "data_0.pkl", 1: BLD / "data_1.pkl"} -``` - -Why does pytask recommend dictionaries and convert lists, tuples, or other -iterators to dictionaries? First, dictionaries with positions as keys behave very -similarly to lists. - -Secondly, dictionary keys are more descriptive and do not assume a fixed -ordering. Both attributes are especially desirable in complex projects. - -**Multiple decorators** - -pytask merges multiple decorators of one kind into a single dictionary. This might help -you to group dependencies and apply them to multiple tasks. - -```python -common_dependencies = pytask.mark.depends_on( - {"first_text": "text_1.txt", "second_text": "text_2.txt"} -) - - -@common_dependencies -@pytask.mark.depends_on("text_3.txt") -def task_example(depends_on): - ... -``` - -Inside the task, `depends_on` will be - -```pycon ->>> depends_on -{"first_text": ... / "text_1.txt", "second_text": "text_2.txt", 0: "text_3.txt"} -``` - -**Nested dependencies and products** - -Dependencies and products can be nested containers consisting of tuples, lists, and -dictionaries. It is beneficial if you want more structure and nesting. - -Here is an example of a task that fits some model on data. It depends on a module -containing the code for the model, which is not actively used but ensures that the task -is rerun when the model is changed. And it depends on the data. - -```python -@pytask.mark.depends_on( - { - "model": [SRC / "models" / "model.py"], - "data": {"a": SRC / "data" / "a.pkl", "b": SRC / "data" / "b.pkl"}, - } -) -@pytask.mark.produces(BLD / "models" / "fitted_model.pkl") -def task_fit_model(depends_on, produces): - ... -``` - -`depends_on` within the function will be - -```python -{ - "model": [SRC / "models" / "model.py"], - "data": {"a": SRC / "data" / "a.pkl", "b": SRC / "data" / "b.pkl"}, -} -``` - ```` ````` diff --git a/docs/source/tutorials/repeating_tasks_with_different_inputs.md b/docs/source/tutorials/repeating_tasks_with_different_inputs.md index 5fed02c7..8e5a10e5 100644 --- a/docs/source/tutorials/repeating_tasks_with_different_inputs.md +++ b/docs/source/tutorials/repeating_tasks_with_different_inputs.md @@ -35,18 +35,6 @@ different seeds and output paths as default arguments of the function. ```{literalinclude} ../../../docs_src/tutorials/repeating_tasks_with_different_inputs1_produces.py ``` -```` - -````{tab-item} Decorators -:sync: decorators - -```{warning} -This approach is deprecated and will be removed in v0.5 -``` - -```{literalinclude} ../../../docs_src/tutorials/repeating_tasks_with_different_inputs1_decorators.py -``` - ```` ````` @@ -83,18 +71,6 @@ You can also add dependencies to repeated tasks just like with any other task. ```{literalinclude} ../../../docs_src/tutorials/repeating_tasks_with_different_inputs2_produces.py ``` -```` - -````{tab-item} Decorators -:sync: decorators - -```{warning} -This approach is deprecated and will be removed in v0.5 -``` - -```{literalinclude} ../../../docs_src/tutorials/repeating_tasks_with_different_inputs2_decorators.py -``` - ```` ````` @@ -155,18 +131,6 @@ For example, the following function is parametrized with tuples. ```{literalinclude} ../../../docs_src/tutorials/repeating_tasks_with_different_inputs3_produces.py ``` -```` - -````{tab-item} Decorators -:sync: decorators - -```{warning} -This approach is deprecated and will be removed in v0.5 -``` - -```{literalinclude} ../../../docs_src/tutorials/repeating_tasks_with_different_inputs3_decorators.py -``` - ```` ````` @@ -208,18 +172,6 @@ a unique name for the iteration. ```{literalinclude} ../../../docs_src/tutorials/repeating_tasks_with_different_inputs4_produces.py ``` -```` - -````{tab-item} Decorators -:sync: decorators - -```{warning} -This approach is deprecated and will be removed in v0.5 -``` - -```{literalinclude} ../../../docs_src/tutorials/repeating_tasks_with_different_inputs4_decorators.py -``` - ```` ````` @@ -306,18 +258,6 @@ Following these three tips, the parametrization becomes ```{literalinclude} ../../../docs_src/tutorials/repeating_tasks_with_different_inputs5_produces.py ``` -```` - -````{tab-item} Decorators -:sync: decorators - -```{warning} -This approach is deprecated and will be removed in v0.5 -``` - -```{literalinclude} ../../../docs_src/tutorials/repeating_tasks_with_different_inputs5_decorators.py -``` - ```` ````` diff --git a/docs/source/tutorials/write_a_task.md b/docs/source/tutorials/write_a_task.md index 878f37cf..8165ffc2 100644 --- a/docs/source/tutorials/write_a_task.md +++ b/docs/source/tutorials/write_a_task.md @@ -94,25 +94,6 @@ the default value of the argument. :emphasize-lines: 9 ``` -```` - -````{tab-item} Decorators - -```{warning} -This approach is deprecated and will be removed in v0.5 -``` - -To specify a product, pass the path to the -{func}`@pytask.mark.produces ` decorator. Then, add `produces` as -an argument name to use the path inside the task function. - -```{literalinclude} ../../../docs_src/tutorials/write_a_task_decorators.py -:emphasize-lines: 10, 11 -``` - -To let pytask track the product of the task, you need to use the -{func}`@pytask.mark.produces ` decorator. - ```` ````` diff --git a/docs_src/tutorials/defining_dependencies_products_dependencies_decorators.py b/docs_src/tutorials/defining_dependencies_products_dependencies_decorators.py deleted file mode 100644 index 3f86a1a0..00000000 --- a/docs_src/tutorials/defining_dependencies_products_dependencies_decorators.py +++ /dev/null @@ -1,18 +0,0 @@ -from pathlib import Path - -import matplotlib.pyplot as plt -import pandas as pd -import pytask -from my_project.config import BLD - - -@pytask.mark.depends_on(BLD / "data.pkl") -@pytask.mark.produces(BLD / "plot.png") -def task_plot_data(depends_on: Path, produces: Path) -> None: - df = pd.read_pickle(depends_on) - - _, ax = plt.subplots() - df.plot(x="x", y="y", ax=ax, kind="scatter") - - plt.savefig(produces) - plt.close() diff --git a/docs_src/tutorials/defining_dependencies_products_products_decorators.py b/docs_src/tutorials/defining_dependencies_products_products_decorators.py deleted file mode 100644 index dbb5fbf0..00000000 --- a/docs_src/tutorials/defining_dependencies_products_products_decorators.py +++ /dev/null @@ -1,20 +0,0 @@ -from pathlib import Path - -import numpy as np -import pandas as pd -import pytask -from my_project.config import BLD - - -@pytask.mark.produces(BLD / "data.pkl") -def task_create_random_data(produces: Path) -> None: - rng = np.random.default_rng(0) - beta = 2 - - x = rng.normal(loc=5, scale=10, size=1_000) - epsilon = rng.standard_normal(1_000) - - y = beta * x + epsilon - - df = pd.DataFrame({"x": x, "y": y}) - df.to_pickle(produces) diff --git a/docs_src/tutorials/defining_dependencies_products_relative_decorators.py b/docs_src/tutorials/defining_dependencies_products_relative_decorators.py deleted file mode 100644 index 0b10e63f..00000000 --- a/docs_src/tutorials/defining_dependencies_products_relative_decorators.py +++ /dev/null @@ -1,8 +0,0 @@ -from pathlib import Path - -import pytask - - -@pytask.mark.produces("../bld/data.pkl") -def task_create_random_data(produces: Path) -> None: - ... diff --git a/docs_src/tutorials/repeating_tasks_with_different_inputs1_decorators.py b/docs_src/tutorials/repeating_tasks_with_different_inputs1_decorators.py deleted file mode 100644 index 0b0890a0..00000000 --- a/docs_src/tutorials/repeating_tasks_with_different_inputs1_decorators.py +++ /dev/null @@ -1,12 +0,0 @@ -from pathlib import Path - -import pytask -from pytask import task - - -for seed in range(10): - - @task - @pytask.mark.produces(Path(f"data_{seed}.pkl")) - def task_create_random_data(produces: Path, seed: int = seed) -> None: - ... diff --git a/docs_src/tutorials/repeating_tasks_with_different_inputs2_decorators.py b/docs_src/tutorials/repeating_tasks_with_different_inputs2_decorators.py deleted file mode 100644 index f332054e..00000000 --- a/docs_src/tutorials/repeating_tasks_with_different_inputs2_decorators.py +++ /dev/null @@ -1,16 +0,0 @@ -from pathlib import Path - -import pytask -from my_project.config import SRC -from pytask import task - - -for seed in range(10): - - @task - @pytask.mark.depends_on(SRC / "parameters.yml") - @pytask.mark.produces(f"data_{seed}.pkl") - def task_create_random_data( - depends_on: Path, produces: Path, seed: int = seed - ) -> None: - ... diff --git a/docs_src/tutorials/repeating_tasks_with_different_inputs3_decorators.py b/docs_src/tutorials/repeating_tasks_with_different_inputs3_decorators.py deleted file mode 100644 index 14b2b404..00000000 --- a/docs_src/tutorials/repeating_tasks_with_different_inputs3_decorators.py +++ /dev/null @@ -1,13 +0,0 @@ -from pathlib import Path -from typing import Tuple - -import pytask -from pytask import task - - -for seed in ((0,), (1,)): - - @task - @pytask.mark.produces(Path(f"data_{seed[0]}.pkl")) - def task_create_random_data(produces: Path, seed: Tuple[int] = seed) -> None: - ... diff --git a/docs_src/tutorials/repeating_tasks_with_different_inputs4_decorators.py b/docs_src/tutorials/repeating_tasks_with_different_inputs4_decorators.py deleted file mode 100644 index 148d89f6..00000000 --- a/docs_src/tutorials/repeating_tasks_with_different_inputs4_decorators.py +++ /dev/null @@ -1,12 +0,0 @@ -from pathlib import Path - -import pytask -from pytask import task - - -for seed, id_ in ((0, "first"), (1, "second")): - - @task(id=id_) - @pytask.mark.produces(Path(f"out_{seed}.txt")) - def task_create_random_data(produces: Path, seed: int = seed) -> None: - ... diff --git a/docs_src/tutorials/repeating_tasks_with_different_inputs5_decorators.py b/docs_src/tutorials/repeating_tasks_with_different_inputs5_decorators.py deleted file mode 100644 index 33091f59..00000000 --- a/docs_src/tutorials/repeating_tasks_with_different_inputs5_decorators.py +++ /dev/null @@ -1,22 +0,0 @@ -from pathlib import Path -from typing import NamedTuple - -from pytask import task - - -class _Arguments(NamedTuple): - seed: int - path_to_data: Path - - -ID_TO_KWARGS = { - "first": _Arguments(seed=0, path_to_data=Path("data_0.pkl")), - "second": _Arguments(seed=1, path_to_data=Path("data_1.pkl")), -} - - -for id_, kwargs in ID_TO_KWARGS.items(): - - @task(id=id_, kwargs=kwargs) - def task_create_random_data(seed: int, produces: Path) -> None: - ... diff --git a/docs_src/tutorials/write_a_task_decorators.py b/docs_src/tutorials/write_a_task_decorators.py deleted file mode 100644 index c7e8c9af..00000000 --- a/docs_src/tutorials/write_a_task_decorators.py +++ /dev/null @@ -1,21 +0,0 @@ -# Content of task_data_preparation.py. -from pathlib import Path - -import numpy as np -import pandas as pd -import pytask -from my_project.config import BLD - - -@pytask.mark.produces(BLD / "data.pkl") -def task_create_random_data(produces: Path) -> None: - rng = np.random.default_rng(0) - beta = 2 - - x = rng.normal(loc=5, scale=10, size=1_000) - epsilon = rng.standard_normal(1_000) - - y = beta * x + epsilon - - df = pd.DataFrame({"x": x, "y": y}) - df.to_pickle(produces) diff --git a/pyproject.toml b/pyproject.toml index 4a42ab3d..61b30ca9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ all = ['universal-pathlib; python_version<"3.12"'] docs = [ "furo", "ipython", + "matplotlib", "myst-parser", "nbsphinx", "sphinx", diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 3b262568..afa1c9bc 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -21,6 +21,7 @@ from _pytask.console import get_file from _pytask.exceptions import CollectionError from _pytask.exceptions import NodeNotCollectedError +from _pytask.mark import MarkGenerator from _pytask.mark_utils import get_all_marks from _pytask.mark_utils import has_mark from _pytask.node_protocols import PNode @@ -235,6 +236,10 @@ def _is_filtered_object(obj: Any) -> bool: See :issue:`507`. """ + # Filter :class:`pytask.mark.MarkGenerator` which can raise errors on some marks. + if isinstance(obj, MarkGenerator): + return True + # Filter :class:`pytask.Task` and :class:`pytask.TaskWithoutPath` objects. if isinstance(obj, PTask) and inspect.isclass(obj): return True diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index 054a01e3..d39bc21b 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -1,35 +1,23 @@ """Contains utility functions for :mod:`_pytask.collect`.""" from __future__ import annotations -import itertools +import inspect import sys -import uuid -import warnings -from pathlib import Path from typing import Any from typing import Callable -from typing import Generator -from typing import Iterable from typing import TYPE_CHECKING import attrs from _pytask._inspect import get_annotations from _pytask.exceptions import NodeNotCollectedError -from _pytask.mark_utils import has_mark -from _pytask.mark_utils import remove_marks from _pytask.models import NodeInfo from _pytask.node_protocols import PNode from _pytask.nodes import PythonNode -from _pytask.shared import find_duplicates from _pytask.task_utils import parse_keyword_arguments_from_signature_defaults -from _pytask.tree_util import PyTree from _pytask.tree_util import tree_leaves -from _pytask.tree_util import tree_map from _pytask.tree_util import tree_map_with_path from _pytask.typing import no_default from _pytask.typing import ProductType -from attrs import define -from attrs import field from typing_extensions import get_origin if sys.version_info >= (3, 9): @@ -38,205 +26,21 @@ from typing_extensions import Annotated if TYPE_CHECKING: + from pathlib import Path from _pytask.session import Session __all__ = [ - "depends_on", "parse_dependencies_from_task_function", - "parse_nodes", "parse_products_from_task_function", - "produces", ] -def depends_on(objects: PyTree[Any]) -> PyTree[Any]: - """Specify dependencies for a task. - - Parameters - ---------- - objects - Can be any valid Python object or an iterable of any Python objects. To be - valid, it must be parsed by some hook implementation for the - :func:`_pytask.hookspecs.pytask_collect_node` entry-point. - - """ - return objects - - -def produces(objects: PyTree[Any]) -> PyTree[Any]: - """Specify products of a task. - - Parameters - ---------- - objects - Can be any valid Python object or an iterable of any Python objects. To be - valid, it must be parsed by some hook implementation for the - :func:`_pytask.hookspecs.pytask_collect_node` entry-point. - - """ - return objects - - -def parse_nodes( # noqa: PLR0913 - session: Session, - task_path: Path | None, - task_name: str, - node_path: Path, - obj: Any, - parser: Callable[..., Any], -) -> Any: - """Parse nodes from object.""" - arg_name = parser.__name__ - objects = _extract_nodes_from_function_markers(obj, parser) - nodes = _convert_objects_to_node_dictionary(objects, arg_name) - return tree_map( - lambda x: _collect_decorator_node( - session, - node_path, - task_name, - NodeInfo( - arg_name=arg_name, - path=(), - value=x, - task_path=task_path, - task_name=task_name, - ), - ), - nodes, - ) - - -def _extract_nodes_from_function_markers( - function: Callable[..., Any], parser: Callable[..., Any] -) -> Generator[Any, None, None]: - """Extract nodes from a marker. - - The parser is a functions which is used to document the marker with the correct - signature. Using the function as a parser for the ``args`` and ``kwargs`` of the - marker provides the expected error message for misspecification. - - """ - marker_name = parser.__name__ - _, markers = remove_marks(function, marker_name) - for marker in markers: - parsed = parser(*marker.args, **marker.kwargs) - yield parsed - - -def _convert_objects_to_node_dictionary(objects: Any, when: str) -> dict[Any, Any]: - """Convert objects to node dictionary.""" - list_of_dicts = [_convert_to_dict(x) for x in objects] - _check_that_names_are_not_used_multiple_times(list_of_dicts, when) - return _merge_dictionaries(list_of_dicts) - - -@define(frozen=True) -class _Placeholder: - """A placeholder to mark unspecified keys in dictionaries.""" - - scalar: bool = field(default=False) - id_: uuid.UUID = field(factory=uuid.uuid4) - - -def _convert_to_dict(x: Any, first_level: bool = True) -> Any | dict[Any, Any]: - """Convert any object to a dictionary.""" - if isinstance(x, dict): - return {k: _convert_to_dict(v, False) for k, v in x.items()} - if isinstance(x, Iterable) and not isinstance(x, str): - if first_level: - return { - _Placeholder(): _convert_to_dict(element, False) - for i, element in enumerate(x) - } - return {i: _convert_to_dict(element, False) for i, element in enumerate(x)} - if first_level: - return {_Placeholder(scalar=True): x} - return x - - -def _check_that_names_are_not_used_multiple_times( - list_of_dicts: list[dict[Any, Any]], when: str -) -> None: - """Check that names of nodes are not assigned multiple times. - - Tuples in the list have either one or two elements. The first element in the two - element tuples is the name and cannot occur twice. - - """ - names_with_provisional_keys = list( - itertools.chain.from_iterable(dict_.keys() for dict_ in list_of_dicts) - ) - names = [x for x in names_with_provisional_keys if not isinstance(x, _Placeholder)] - duplicated = find_duplicates(names) - - if duplicated: - msg = f"'@pytask.mark.{when}' has nodes with the same name: {duplicated}" - raise ValueError(msg) - - -def _union_of_dictionaries(dicts: list[dict[Any, Any]]) -> dict[Any, Any]: - """Merge multiple dictionaries in one. - - Examples - -------- - >>> a, b = {"a": 0}, {"b": 1} - >>> _union_of_dictionaries([a, b]) - {'a': 0, 'b': 1} - - >>> a, b = {'a': 0}, {'a': 1} - >>> _union_of_dictionaries([a, b]) - {'a': 1} - - """ - return dict(itertools.chain.from_iterable(dict_.items() for dict_ in dicts)) - - -def _merge_dictionaries(list_of_dicts: list[dict[Any, Any]]) -> dict[Any, Any]: - """Merge multiple dictionaries. - - The function does not perform a deep merge. It simply merges the dictionary based on - the first level keys which are either unique names or placeholders. During the merge - placeholders will be replaced by an incrementing integer. - - Examples - -------- - >>> a, b = {"a": 0}, {"b": 1} - >>> _merge_dictionaries([a, b]) - {'a': 0, 'b': 1} - - >>> a, b = {_Placeholder(): 0}, {_Placeholder(): 1} - >>> _merge_dictionaries([a, b]) - {0: 0, 1: 1} - - """ - merged_dict = _union_of_dictionaries(list_of_dicts) - - if len(merged_dict) == 1 and isinstance(next(iter(merged_dict)), _Placeholder): - placeholder, value = next(iter(merged_dict.items())) - out = value if placeholder.scalar else {0: value} - else: - counter = itertools.count() - out = {} - for k, v in merged_dict.items(): - if isinstance(k, _Placeholder): - while True: - possible_key = next(counter) - if possible_key not in merged_dict and possible_key not in out: - out[possible_key] = v - break - else: - out[k] = v - - return out - - _ERROR_MULTIPLE_DEPENDENCY_DEFINITIONS = """The task uses multiple ways to define \ dependencies. Dependencies should be defined with either - as default value for the function argument 'depends_on'. - as '@pytask.task(kwargs={"depends_on": ...})' -- or with the deprecated '@pytask.mark.depends_on' and a 'depends_on' function argument. Use only one of the two ways! @@ -250,27 +54,13 @@ def parse_dependencies_from_task_function( session: Session, task_path: Path | None, task_name: str, node_path: Path, obj: Any ) -> dict[str, Any]: """Parse dependencies from task function.""" - has_depends_on_decorator = False - has_depends_on_argument = False dependencies = {} - if has_mark(obj, "depends_on"): - has_depends_on_decorator = True - nodes = parse_nodes(session, task_path, task_name, node_path, obj, depends_on) - dependencies["depends_on"] = nodes - task_kwargs = obj.pytask_meta.kwargs if hasattr(obj, "pytask_meta") else {} signature_defaults = parse_keyword_arguments_from_signature_defaults(obj) kwargs = {**signature_defaults, **task_kwargs} kwargs.pop("produces", None) - # Parse dependencies from task when @task is used. - if "depends_on" in kwargs: - has_depends_on_argument = True - - if has_depends_on_decorator and has_depends_on_argument: - raise NodeNotCollectedError(_ERROR_MULTIPLE_DEPENDENCY_DEFINITIONS) - parameters_with_product_annot = _find_args_with_product_annotation(obj) parameters_with_node_annot = _find_args_with_node_annotation(obj) @@ -326,7 +116,7 @@ def parse_dependencies_from_task_function( ) dependencies[parameter_name] = PythonNode(value=value, name=node_name) else: - dependencies[parameter_name] = nodes + dependencies[parameter_name] = nodes # type: ignore[assignment] return dependencies @@ -354,19 +144,23 @@ def _find_args_with_node_annotation(func: Callable[..., Any]) -> dict[str, PNode return args_with_node_annotation -_ERROR_MULTIPLE_PRODUCT_DEFINITIONS = """The task uses multiple ways to define \ -products. Products should be defined with either +_ERROR_MULTIPLE_TASK_RETURN_DEFINITIONS = """The task uses multiple ways to parse \ +products from the return of the task function. Use either + +def task_example() -> Annotated[str, Path("file.txt")]: + ... + +or -- 'typing.Annotated[Path, Product] = Path(...)' (recommended) -- '@pytask.mark.task(kwargs={'produces': Path(...)})' -- as a default argument for 'produces': 'produces = Path(...)' -- '@pytask.mark.produces(Path(...))' (deprecated) +@task(produces=Path("file.txt")) +def task_example() -> str: + ... -Read more about products in the documentation: https://tinyurl.com/pytask-deps-prods. +Read more about products in the documentation: http://tinyurl.com/pytask-return. """ -def parse_products_from_task_function( # noqa: C901 +def parse_products_from_task_function( session: Session, task_path: Path | None, task_name: str, node_path: Path, obj: Any ) -> dict[str, Any]: """Parse products from task function. @@ -374,37 +168,26 @@ def parse_products_from_task_function( # noqa: C901 Raises ------ NodeNotCollectedError - If multiple ways were used to specify products. + If multiple ways to parse products from the return of the task function are + used. """ - has_produces_decorator = False - has_produces_argument = False - has_annotation = False has_return = False has_task_decorator = False - out = {} - # Parse products from decorators. - if has_mark(obj, "produces"): - has_produces_decorator = True - nodes = parse_nodes(session, task_path, task_name, node_path, obj, produces) - out = {"produces": nodes} + out: dict[str, Any] = {} task_kwargs = obj.pytask_meta.kwargs if hasattr(obj, "pytask_meta") else {} signature_defaults = parse_keyword_arguments_from_signature_defaults(obj) kwargs = {**signature_defaults, **task_kwargs} + parameters = list(inspect.signature(obj).parameters) parameters_with_product_annot = _find_args_with_product_annotation(obj) parameters_with_node_annot = _find_args_with_node_annotation(obj) # Allow to collect products from 'produces'. - if "produces" in kwargs: - if "produces" not in parameters_with_product_annot: - parameters_with_product_annot.append("produces") - # If there are more parameters with a product annotation, we want to raise an - # error later to warn about mixing different interfaces. - if set(parameters_with_product_annot) - {"produces"}: - has_produces_argument = True + if "produces" in parameters and "produces" not in parameters_with_product_annot: + parameters_with_product_annot.append("produces") if "return" in parameters_with_node_annot: parameters_with_product_annot.append("return") @@ -413,9 +196,8 @@ def parse_products_from_task_function( # noqa: C901 if parameters_with_product_annot: out = {} for parameter_name in parameters_with_product_annot: - if parameter_name != "return": - has_annotation = True - + # Makes sure that missing products will show up as missing inputs during the + # execution. if ( parameter_name not in kwargs and parameter_name not in parameters_with_node_annot @@ -474,19 +256,8 @@ def parse_products_from_task_function( # noqa: C901 ) out = {"return": collected_products} - if ( - sum( - ( - has_produces_decorator, - has_produces_argument, - has_annotation, - has_return, - has_task_decorator, - ) - ) - >= 2 # noqa: PLR2004 - ): - raise NodeNotCollectedError(_ERROR_MULTIPLE_PRODUCT_DEFINITIONS) + if sum((has_return, has_task_decorator)) == 2: # noqa: PLR2004 + raise NodeNotCollectedError(_ERROR_MULTIPLE_TASK_RETURN_DEFINITIONS) return out @@ -509,56 +280,6 @@ def _find_args_with_product_annotation(func: Callable[..., Any]) -> list[str]: return args_with_product_annot -_ERROR_WRONG_TYPE_DECORATOR = """'@pytask.mark.depends_on', '@pytask.mark.produces', \ -and their function arguments can only accept values of type 'str' and 'pathlib.Path' \ -or the same values nested in tuples, lists, and dictionaries. Here, {node} has type \ -{node_type}. -""" - - -_WARNING_STRING_DEPRECATED = """Using strings to specify a {kind} is deprecated. Pass \ -a 'pathlib.Path' instead with 'Path("{node}")'. -""" - - -def _collect_decorator_node( - session: Session, path: Path, name: str, node_info: NodeInfo -) -> PNode: - """Collect nodes for a task. - - Raises - ------ - NodeNotCollectedError - If the node could not collected. - - """ - node = node_info.value - kind = {"depends_on": "dependency", "produces": "product"}.get(node_info.arg_name) - - if not isinstance(node, (str, Path)): - raise NodeNotCollectedError( - _ERROR_WRONG_TYPE_DECORATOR.format(node=node, node_type=type(node)) - ) - - if isinstance(node, str): - warnings.warn( - _WARNING_STRING_DEPRECATED.format(kind=kind, node=node), - category=FutureWarning, - stacklevel=1, - ) - node = Path(node) - node_info = node_info._replace(value=node) - - collected_node = session.hook.pytask_collect_node( - session=session, path=path, node_info=node_info - ) - if collected_node is None: # pragma: no cover - msg = f"{node!r} cannot be parsed as a {kind} for task {name!r} in {path!r}." - raise NodeNotCollectedError(msg) - - return collected_node - - def _collect_dependency( session: Session, path: Path, name: str, node_info: NodeInfo ) -> PNode: @@ -572,16 +293,6 @@ def _collect_dependency( """ node = node_info.value - # If we encounter a string and the argument name is 'depends_on', we convert it. - if isinstance(node, str) and node_info.arg_name == "depends_on": - warnings.warn( - _WARNING_STRING_DEPRECATED.format(kind="dependency", node=node), - category=FutureWarning, - stacklevel=1, - ) - node = Path(node) - node_info = node_info._replace(value=node) - if isinstance(node, PythonNode) and node.value is no_default: # If a node is a dependency and its value is not set, the node is a product in # another task and the value will be set there. Thus, we wrap the original node @@ -619,16 +330,6 @@ def _collect_product( """ node = node_info.value - # If we encounter a string and the argument name is 'produces', we convert it. - if isinstance(node, str) and node_info.arg_name == "produces": - warnings.warn( - _WARNING_STRING_DEPRECATED.format(kind="product", node=node), - category=FutureWarning, - stacklevel=1, - ) - node = Path(node) - node_info = node_info._replace(value=node) - collected_node = session.hook.pytask_collect_node( session=session, path=path, node_info=node_info ) diff --git a/src/_pytask/config.py b/src/_pytask/config.py index ef176100..d42d5eed 100644 --- a/src/_pytask/config.py +++ b/src/_pytask/config.py @@ -80,14 +80,6 @@ def pytask_parse_config(config: dict[str, Any]) -> None: config["paths"] = parse_paths(config["paths"]) config["markers"] = { - "depends_on": ( - "Add dependencies to a task. See this tutorial for more information: " - "[link https://bit.ly/3JlxylS]https://bit.ly/3JlxylS[/]." - ), - "produces": ( - "Add products to a task. See this tutorial for more information: " - "[link https://bit.ly/3JlxylS]https://bit.ly/3JlxylS[/]." - ), "try_first": "Try to execute a task a early as possible.", "try_last": "Try to execute a task a late as possible.", **config["markers"], diff --git a/src/_pytask/mark/__init__.pyi b/src/_pytask/mark/__init__.pyi index f9d55332..3b25e360 100644 --- a/src/_pytask/mark/__init__.pyi +++ b/src/_pytask/mark/__init__.pyi @@ -1,4 +1,3 @@ -from pathlib import Path from typing import Any from typing_extensions import deprecated from _pytask.mark.expression import Expression @@ -15,20 +14,6 @@ def select_by_keyword(session: Session, dag: nx.DiGraph) -> set[str]: ... def select_by_mark(session: Session, dag: nx.DiGraph) -> set[str]: ... class MarkGenerator: - @deprecated( - "'@pytask.mark.produces' is deprecated starting pytask v0.4.0 and will be removed in v0.5.0. To upgrade your project to the new syntax, read the tutorial on product and dependencies: https://tinyurl.com/pytask-deps-prods.", # noqa: E501 - category=FutureWarning, - stacklevel=1, - ) - @staticmethod - def produces(objects: PyTree[str | Path]) -> None: ... - @deprecated( - "'@pytask.mark.depends_on' is deprecated starting pytask v0.4.0 and will be removed in v0.5.0. To upgrade your project to the new syntax, read the tutorial on product and dependencies: https://tinyurl.com/pytask-deps-prods.", # noqa: E501 - category=FutureWarning, - stacklevel=1, - ) - @staticmethod - def depends_on(objects: PyTree[str | Path]) -> None: ... @deprecated( "'@pytask.mark.task' is deprecated starting pytask v0.4.0 and will be removed in v0.5.0. Use '@task' from 'from pytask import task' instead.", # noqa: E501 category=FutureWarning, diff --git a/src/_pytask/mark/structures.py b/src/_pytask/mark/structures.py index 126591ca..1efbc8c2 100644 --- a/src/_pytask/mark/structures.py +++ b/src/_pytask/mark/structures.py @@ -162,9 +162,9 @@ def store_mark(obj: Callable[..., Any], mark: Mark) -> None: ) -_DEPRECATION_DECORATOR = """'@pytask.mark.{}' is deprecated starting pytask \ -v0.4.0 and will be removed in v0.5.0. To upgrade your project to the new syntax, read \ -the tutorial on product and dependencies: https://tinyurl.com/pytask-deps-prods. +_DEPRECATION_DECORATOR = """'@pytask.mark.{}' was removed in pytask v0.5.0. To upgrade \ +your project to the new syntax, read the tutorial on product and dependencies: \ +https://tinyurl.com/pytask-deps-prods. """ @@ -194,11 +194,7 @@ def __getattr__(self, name: str) -> MarkDecorator | Any: raise AttributeError(msg) if name in ("depends_on", "produces"): - warnings.warn( - _DEPRECATION_DECORATOR.format(name), - category=FutureWarning, - stacklevel=1, - ) + raise RuntimeError(_DEPRECATION_DECORATOR.format(name)) # If the name is not in the set of known marks after updating, # then it really is time to issue a warning or an error. diff --git a/src/pytask/__init__.py b/src/pytask/__init__.py index 8c530cc2..d8ed8131 100644 --- a/src/pytask/__init__.py +++ b/src/pytask/__init__.py @@ -9,10 +9,8 @@ from _pytask.click import ColoredCommand from _pytask.click import ColoredGroup from _pytask.click import EnumChoice -from _pytask.collect_utils import depends_on from _pytask.collect_utils import parse_dependencies_from_task_function from _pytask.collect_utils import parse_products_from_task_function -from _pytask.collect_utils import produces from _pytask.compat import check_for_optional_program from _pytask.compat import import_optional_dependency from _pytask.console import console @@ -135,7 +133,6 @@ "console", "count_outcomes", "create_database", - "depends_on", "get_all_marks", "get_marks", "get_plugin_manager", @@ -148,7 +145,6 @@ "parse_dependencies_from_task_function", "parse_products_from_task_function", "parse_warning_filter", - "produces", "remove_internal_traceback_frames_from_exc_info", "remove_marks", "set_marks", diff --git a/tests/test_build.py b/tests/test_build.py index 96f158e6..526079bb 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -39,17 +39,10 @@ def test_collection_failed(runner, tmp_path): @pytest.mark.end_to_end() def test_building_dag_failed(runner, tmp_path): source = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("in.txt") - @pytask.mark.produces("out.txt") - def task_passes_1(): - pass - - @pytask.mark.depends_on("out.txt") - @pytask.mark.produces("in.txt") - def task_passes_2(): - pass + def task_passes_1(in_path = Path("in.txt"), produces = Path("out.txt")): ... + def task_passes_2(in_path = Path("out.txt"), produces = Path("in.txt")): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) diff --git a/tests/test_clean.py b/tests/test_clean.py index 91c59f0e..0dd39611 100644 --- a/tests/test_clean.py +++ b/tests/test_clean.py @@ -14,10 +14,9 @@ _PROJECT_TASK = """ import pytask +from pathlib import Path -@pytask.mark.depends_on("in.txt") -@pytask.mark.produces("out.txt") -def task_write_text(depends_on, produces): +def task_write_text(path = Path("in.txt"), produces = Path("out.txt")): produces.write_text("a") """ @@ -26,7 +25,7 @@ def task_write_text(depends_on, produces): import pytask from pathlib import Path -def task_write_text(in_path=Path("in.txt"), produces=Path("out.txt")): +def task_write_text(path=Path("in.txt"), produces=Path("out.txt")): produces.write_text("a") """ @@ -46,10 +45,9 @@ def project(request, tmp_path): _GIT_PROJECT_TASK = """ import pytask +from pathlib import Path -@pytask.mark.depends_on("in_tracked.txt") -@pytask.mark.produces("out.txt") -def task_write_text(depends_on, produces): +def task_write_text(path = Path("in_tracked.txt"), produces = Path("out.txt")): produces.write_text("a") """ @@ -58,7 +56,7 @@ def task_write_text(depends_on, produces): import pytask from pathlib import Path -def task_write_text(in_path=Path("in_tracked.txt"), produces=Path("out.txt")): +def task_write_text(path=Path("in_tracked.txt"), produces=Path("out.txt")): produces.write_text("a") """ diff --git a/tests/test_collect.py b/tests/test_collect.py index 6d01eec6..e9c928a8 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -13,7 +13,6 @@ from pytask import CollectionOutcome from pytask import ExitCode from pytask import NodeInfo -from pytask import NodeNotCollectedError from pytask import Session from pytask import Task @@ -28,13 +27,10 @@ ) def test_collect_file_with_relative_path(tmp_path, depends_on, produces): source = f""" - import pytask from pathlib import Path - @pytask.mark.depends_on({depends_on}) - @pytask.mark.produces({produces}) - def task_write_text(depends_on, produces): - produces.write_text(depends_on.read_text()) + def task_write_text(path=Path({depends_on}), produces=Path({produces})): + produces.write_text(path.read_text()) """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("Relative paths work.") @@ -66,67 +62,17 @@ def task_example( assert tmp_path.joinpath("out.txt").exists() -@pytest.mark.end_to_end() -def test_collect_depends_on_that_is_not_str_or_path(capsys, tmp_path): - """If a node cannot be parsed because unknown type, raise an error.""" - source = """ - import pytask - - @pytask.mark.depends_on(True) - def task_with_non_path_dependency(): - pass - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - session = build(paths=tmp_path) - - assert session.exit_code == ExitCode.COLLECTION_FAILED - assert session.collection_reports[0].outcome == CollectionOutcome.FAIL - exc_info = session.collection_reports[0].exc_info - assert isinstance(exc_info[1], NodeNotCollectedError) - captured = capsys.readouterr().out - assert "'@pytask.mark.depends_on'" in captured - # Assert tracebacks are hidden. - assert "_pytask/collect.py" not in captured - - -@pytest.mark.end_to_end() -def test_collect_produces_that_is_not_str_or_path(tmp_path, capsys): - """If a node cannot be parsed because unknown type, raise an error.""" - source = """ - import pytask - - @pytask.mark.produces(True) - def task_with_non_path_dependency(): - pass - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - session = build(paths=tmp_path) - - assert session.exit_code == ExitCode.COLLECTION_FAILED - assert session.collection_reports[0].outcome == CollectionOutcome.FAIL - exc_info = session.collection_reports[0].exc_info - assert isinstance(exc_info[1], NodeNotCollectedError) - captured = capsys.readouterr().out - assert "'@pytask.mark.depends_on'" in captured - - @pytest.mark.end_to_end() def test_collect_nodes_with_the_same_name(runner, tmp_path): """Nodes with the same filename, not path, are not mistaken for each other.""" source = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("text.txt") - @pytask.mark.produces("out_0.txt") - def task_0(depends_on, produces): - produces.write_text(depends_on.read_text()) + def task_0(path=Path("text.txt"), produces=Path("out_0.txt")): + produces.write_text(path.read_text()) - @pytask.mark.depends_on("sub/text.txt") - @pytask.mark.produces("out_1.txt") - def task_1(depends_on, produces): - produces.write_text(depends_on.read_text()) + def task_1(path=Path("sub/text.txt"), produces=Path("out_1.txt")): + produces.write_text(path.read_text()) """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -373,25 +319,6 @@ def task_my_task(): assert outcome == CollectionOutcome.SUCCESS -@pytest.mark.end_to_end() -@pytest.mark.parametrize("decorator", ["", "@task"]) -def test_collect_string_product_with_or_without_task_decorator( - runner, tmp_path, decorator -): - source = f""" - from pytask import task - - {decorator} - def task_write_text(produces="out.txt"): - produces.touch() - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == ExitCode.OK - assert tmp_path.joinpath("out.txt").exists() - assert "FutureWarning" in result.output - - @pytest.mark.end_to_end() def test_collect_string_product_raises_error_with_annotation(runner, tmp_path): """The string is not converted to a path.""" @@ -407,98 +334,6 @@ def task_write_text(out: Annotated[str, Product] = "out.txt") -> None: assert result.exit_code == ExitCode.FAILED -@pytest.mark.end_to_end() -def test_product_cannot_mix_different_product_types(tmp_path, capsys): - source = """ - import pytask - from typing_extensions import Annotated - from pytask import Product - from pathlib import Path - - @pytask.mark.produces("out_deco.txt") - def task_example( - path: Annotated[Path, Product], produces: Path = Path("out_sig.txt") - ): - ... - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build(paths=tmp_path) - - assert session.exit_code == ExitCode.COLLECTION_FAILED - assert len(session.tasks) == 0 - report = session.collection_reports[0] - assert report.outcome == CollectionOutcome.FAIL - captured = capsys.readouterr().out - assert "The task uses multiple ways" in captured - - -@pytest.mark.end_to_end() -def test_depends_on_cannot_mix_different_definitions(tmp_path, capsys): - source = """ - import pytask - from typing_extensions import Annotated - from pytask import Product - from pathlib import Path - - @pytask.mark.depends_on("input_1.txt") - def task_example( - depends_on: Path = "input_2.txt", - path: Annotated[Path, Product] = Path("out.txt") - ): - ... - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - tmp_path.joinpath("input_1.txt").touch() - tmp_path.joinpath("input_2.txt").touch() - session = build(paths=tmp_path) - - assert session.exit_code == ExitCode.COLLECTION_FAILED - assert len(session.tasks) == 0 - report = session.collection_reports[0] - assert report.outcome == CollectionOutcome.FAIL - captured = capsys.readouterr().out - assert "The task uses multiple" in captured - - -@pytest.mark.end_to_end() -@pytest.mark.parametrize( - ("depends_on", "produces"), - [("'in.txt'", "Path('out.txt')"), ("Path('in.txt')", "'out.txt'")], -) -def test_deprecation_warning_for_strings_in_former_decorator_args( - runner, tmp_path, depends_on, produces -): - source = f""" - import pytask - from pathlib import Path - - @pytask.mark.depends_on({depends_on}) - @pytask.mark.produces({produces}) - def task_write_text(depends_on, produces): - produces.touch() - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - tmp_path.joinpath("in.txt").touch() - - result = runner.invoke(cli, [tmp_path.as_posix()]) - assert "FutureWarning" in result.output - - -@pytest.mark.end_to_end() -def test_no_deprecation_warning_for_using_magic_produces(runner, tmp_path): - source = """ - import pytask - from pathlib import Path - - def task_write_text(depends_on, produces=Path("out.txt")): - produces.touch() - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - result = runner.invoke(cli, [tmp_path.as_posix()]) - assert "FutureWarning" not in result.output - - @pytest.mark.end_to_end() def test_setting_name_for_path_node_via_annotation(tmp_path): source = """ @@ -522,13 +357,11 @@ def task_example( @pytest.mark.end_to_end() def test_error_when_dependency_is_defined_in_kwargs_and_annotation(runner, tmp_path): source = """ - import pytask from pathlib import Path from typing_extensions import Annotated - from pytask import Product, PathNode - from pytask import PythonNode + from pytask import Product, PathNode, PythonNode, task - @pytask.mark.task(kwargs={"in_": "world"}) + @task(kwargs={"in_": "world"}) def task_example( in_: Annotated[str, PythonNode(name="string", value="hello")], path: Annotated[Path, Product, PathNode(path=Path("out.txt"), name="product")], @@ -544,14 +377,13 @@ def task_example( @pytest.mark.end_to_end() def test_error_when_product_is_defined_in_kwargs_and_annotation(runner, tmp_path): source = """ - import pytask from pathlib import Path from typing_extensions import Annotated - from pytask import Product, PathNode + from pytask import Product, PathNode, task node = PathNode(path=Path("out.txt"), name="product") - @pytask.mark.task(kwargs={"path": node}) + @task(kwargs={"path": node}) def task_example(path: Annotated[Path, Product, node]) -> None: path.write_text("text") """ @@ -713,3 +545,20 @@ def task_example( result = runner.invoke(cli, [tmp_path.as_posix()]) assert result.exit_code == ExitCode.COLLECTION_FAILED assert "Parameter 'dependency' has multiple node annot" in result.output + + +@pytest.mark.end_to_end() +def test_error_if_multiple_return_annotations_are_used(runner, tmp_path): + source = """ + from pytask import task + from pathlib import Path + from typing_extensions import Annotated + + @task(produces=Path("file.txt")) + def task_example() -> Annotated[str, Path("file.txt")]: ... + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.COLLECTION_FAILED + assert "The task uses multiple ways to parse" in result.output diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py index 94788da8..66cf9b97 100644 --- a/tests/test_collect_command.py +++ b/tests/test_collect_command.py @@ -19,12 +19,9 @@ @pytest.mark.end_to_end() def test_collect_task(runner, tmp_path): source = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("in.txt") - @pytask.mark.produces("out.txt") - def task_example(): - pass + def task_example(path=Path("in.txt"), produces=Path("out.txt")): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").touch() @@ -55,11 +52,9 @@ def task_example(): @pytest.mark.end_to_end() def test_collect_task_new_interface(runner, tmp_path): source = """ - import pytask from pathlib import Path - def task_example(depends_on=Path("in.txt"), arg=1, produces=Path("out.txt")): - pass + def task_example(depends_on=Path("in.txt"), arg=1, produces=Path("out.txt")): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").touch() @@ -91,12 +86,9 @@ def task_example(depends_on=Path("in.txt"), arg=1, produces=Path("out.txt")): @pytest.mark.end_to_end() def test_collect_task_in_root_dir(runner, tmp_path): source = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("in.txt") - @pytask.mark.produces("out.txt") - def task_example(): - pass + def task_example(path=Path("in.txt"), produces=Path("out.txt")): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").touch() @@ -117,13 +109,13 @@ def task_example(): @pytest.mark.end_to_end() def test_collect_parametrized_tasks(runner, tmp_path): source = """ - import pytask + from pytask import task + from pathlib import Path for arg, produces in [(0, "out_0.txt"), (1, "out_1.txt")]: - @pytask.mark.task - @pytask.mark.depends_on("in.txt") - def task_example(arg=arg, produces=produces): + @task + def task_example(depends_on=Path("in.txt"), arg=arg, produces=produces): pass """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -136,24 +128,17 @@ def task_example(arg=arg, produces=produces): assert "" in captured assert "" in captured - assert "[1-out_1.txt]>" in captured + assert "[depends_on0-0-out_0.txt]>" in captured + assert "[depends_on1-1-out_1.txt]>" in captured @pytest.mark.end_to_end() def test_collect_task_with_expressions(runner, tmp_path): source = """ - import pytask - - @pytask.mark.depends_on("in_1.txt") - @pytask.mark.produces("out_1.txt") - def task_example_1(): - pass + from pathlib import Path - @pytask.mark.depends_on("in_2.txt") - @pytask.mark.produces("out_2.txt") - def task_example_2(): - pass + def task_example_1(path=Path("in_1.txt"), produces=Path("out_1.txt")): ... + def task_example_2(path=Path("in_2.txt"), produces=Path("out_2.txt")): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in_1.txt").touch() @@ -188,17 +173,12 @@ def task_example_2(): def test_collect_task_with_marker(runner, tmp_path): source = """ import pytask + from pathlib import Path @pytask.mark.wip - @pytask.mark.depends_on("in_1.txt") - @pytask.mark.produces("out_1.txt") - def task_example_1(): - pass - - @pytask.mark.depends_on("in_2.txt") - @pytask.mark.produces("out_2.txt") - def task_example_2(): - pass + def task_example_1(path=Path("in_1.txt"), produces=Path("out_1.txt")): ... + + def task_example_2(path=Path("in_2.txt"), produces=Path("out_2.txt")): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in_1.txt").touch() @@ -239,20 +219,14 @@ def task_example_2(): @pytest.mark.end_to_end() def test_collect_task_with_ignore_from_config(runner, tmp_path): source = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("in_1.txt") - @pytask.mark.produces("out_1.txt") - def task_example_1(): - pass + def task_example_1(path=Path("in_1.txt"), produces=Path("out_1.txt")): ... """ tmp_path.joinpath("task_example_1.py").write_text(textwrap.dedent(source)) source = """ - @pytask.mark.depends_on("in_2.txt") - @pytask.mark.produces("out_2.txt") - def task_example_2(): - pass + def task_example_2(path=Path("in_2.txt"), produces=Path("out_2.txt")): ... """ tmp_path.joinpath("task_example_2.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in_1.txt").touch() @@ -293,21 +267,17 @@ def task_example_2(): @pytest.mark.end_to_end() def test_collect_task_with_ignore_from_cli(runner, tmp_path): source = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("in_1.txt") - @pytask.mark.produces("out_1.txt") - def task_example_1(): - pass + def task_example_1(path=Path("in_1.txt"), produces=Path("out_1.txt")): ... """ tmp_path.joinpath("task_example_1.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in_1.txt").touch() source = """ - @pytask.mark.depends_on("in_2.txt") - @pytask.mark.produces("out_2.txt") - def task_example_2(): - pass + from pathlib import Path + + def task_example_2(path=Path("in_2.txt"), produces=Path("out_2.txt")): ... """ tmp_path.joinpath("task_example_2.py").write_text(textwrap.dedent(source)) diff --git a/tests/test_collect_utils.py b/tests/test_collect_utils.py index 55ee009d..aabe7036 100644 --- a/tests/test_collect_utils.py +++ b/tests/test_collect_utils.py @@ -1,167 +1,11 @@ from __future__ import annotations -import itertools -from contextlib import ExitStack as does_not_raise # noqa: N813 - -import pytask import pytest -from _pytask.collect_utils import _check_that_names_are_not_used_multiple_times -from _pytask.collect_utils import _convert_objects_to_node_dictionary -from _pytask.collect_utils import _convert_to_dict -from _pytask.collect_utils import _extract_nodes_from_function_markers from _pytask.collect_utils import _find_args_with_product_annotation -from _pytask.collect_utils import _merge_dictionaries -from _pytask.collect_utils import _Placeholder -from pytask import depends_on -from pytask import produces -from pytask import Product +from pytask import Product # noqa: TCH002 from typing_extensions import Annotated -ERROR = "'@pytask.mark.depends_on' has nodes with the same name:" - - -@pytest.mark.unit() -@pytest.mark.parametrize( - ("x", "expectation"), - [ - ([{0: "a"}, {0: "b"}], pytest.raises(ValueError, match=ERROR)), - ([{"a": 0}, {"a": 1}], pytest.raises(ValueError, match=ERROR)), - ([{"a": 0}, {"b": 0}, {"a": 1}], pytest.raises(ValueError, match=ERROR)), - ([{0: "a"}, {1: "a"}], does_not_raise()), - ([{"a": 0}, {0: "a"}], does_not_raise()), - ([{"a": 0}, {"b": 1}], does_not_raise()), - ], -) -def test_check_that_names_are_not_used_multiple_times(x, expectation): - with expectation: - _check_that_names_are_not_used_multiple_times(x, "depends_on") - - -@pytest.mark.integration() -@pytest.mark.parametrize("when", ["depends_on", "produces"]) -@pytest.mark.parametrize( - ("objects", "expectation", "expected"), - [ - ([0, 1], does_not_raise, {0: 0, 1: 1}), - ([{0: 0}, {1: 1}], does_not_raise, {0: 0, 1: 1}), - ([{0: 0}], does_not_raise, {0: 0}), - ([[0]], does_not_raise, {0: 0}), - ( - [((0, 0),), ((0, 1),)], - does_not_raise, - {0: {0: 0, 1: 0}, 1: {0: 0, 1: 1}}, - ), - ([{0: {0: {0: 0}}}, [2]], does_not_raise, {0: {0: {0: 0}}, 1: 2}), - ([{0: 0}, {0: 1}], ValueError, None), - ], -) -def test_convert_objects_to_node_dictionary(objects, when, expectation, expected): - expectation = ( - pytest.raises(expectation, match=f"'@pytask.mark.{when}' has nodes") - if expectation == ValueError - else expectation() - ) - with expectation: - nodes = _convert_objects_to_node_dictionary(objects, when) - assert nodes == expected - - -def _convert_placeholders_to_tuples(x): - counter = itertools.count() - return { - (next(counter), k.scalar) - if isinstance(k, _Placeholder) - else k: _convert_placeholders_to_tuples(v) if isinstance(v, dict) else v - for k, v in x.items() - } - - -@pytest.mark.unit() -@pytest.mark.parametrize( - ("x", "first_level", "expected"), - [ - (1, True, {(0, True): 1}), - ({1: 0}, False, {1: 0}), - ({1: [2, 3]}, False, {1: {0: 2, 1: 3}}), - ([2, 3], True, {(0, False): 2, (1, False): 3}), - ([2, 3], False, {0: 2, 1: 3}), - ], -) -def test__convert_to_dict(x, first_level, expected): - """We convert placeholders to a tuple consisting of the key and the scalar bool.""" - result = _convert_to_dict(x, first_level) - modified_result = _convert_placeholders_to_tuples(result) - assert modified_result == expected - - -@pytest.mark.unit() -@pytest.mark.parametrize( - ("list_of_dicts", "expected"), - [ - ([{1: 0}, {0: 1}], {1: 0, 0: 1}), - ([{_Placeholder(): 1}, {0: 0}], {1: 1, 0: 0}), - ([{_Placeholder(scalar=True): 1}], 1), - ([{_Placeholder(scalar=False): 1}], {0: 1}), - ], -) -def test_merge_dictionaries(list_of_dicts, expected): - result = _merge_dictionaries(list_of_dicts) - assert result == expected - - -@pytest.mark.unit() -@pytest.mark.parametrize("decorator", [pytask.mark.depends_on, pytask.mark.produces]) -@pytest.mark.parametrize( - ("values", "expected"), [("a", ["a"]), (["b"], [["b"]]), (["e", "f"], [["e", "f"]])] -) -def test_extract_args_from_mark(decorator, values, expected): - @decorator(values) - def task_example(): # pragma: no cover - pass - - parser = depends_on if decorator.name == "depends_on" else produces - result = list(_extract_nodes_from_function_markers(task_example, parser)) - assert result == expected - - -@pytest.mark.unit() -@pytest.mark.parametrize("decorator", [pytask.mark.depends_on, pytask.mark.produces]) -@pytest.mark.parametrize( - ("values", "expected"), - [ - ({"objects": "a"}, ["a"]), - ({"objects": ["b"]}, [["b"]]), - ({"objects": ["e", "f"]}, [["e", "f"]]), - ], -) -def test_extract_kwargs_from_mark(decorator, values, expected): - @decorator(**values) - def task_example(): # pragma: no cover - pass - - parser = depends_on if decorator.name == "depends_on" else produces - result = list(_extract_nodes_from_function_markers(task_example, parser)) - assert result == expected - - -@pytest.mark.unit() -@pytest.mark.parametrize("decorator", [pytask.mark.depends_on, pytask.mark.produces]) -@pytest.mark.parametrize( - ("args", "kwargs"), [(["a", "b"], {"objects": "a"}), (("a"), {"objects": "a"})] -) -def test_raise_error_for_invalid_args_to_depends_on_and_produces( - decorator, args, kwargs -): - @decorator(*args, **kwargs) - def task_example(): # pragma: no cover - pass - - parser = depends_on if decorator.name == "depends_on" else produces - with pytest.raises(TypeError): - list(_extract_nodes_from_function_markers(task_example, parser)) - - @pytest.mark.unit() def test_find_args_with_product_annotation(): def func( diff --git a/tests/test_dag.py b/tests/test_dag.py index cbdfae7e..f801b7cd 100644 --- a/tests/test_dag.py +++ b/tests/test_dag.py @@ -41,16 +41,12 @@ def test_pytask_dag_create_dag(): @pytest.mark.end_to_end() def test_cycle_in_dag(tmp_path, runner, snapshot_cli): source = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("out_2.txt") - @pytask.mark.produces("out_1.txt") - def task_1(produces): + def task_1(path = Path("out_2.txt"), produces = Path("out_1.txt")): produces.write_text("1") - @pytask.mark.depends_on("out_1.txt") - @pytask.mark.produces("out_2.txt") - def task_2(produces): + def task_2(path = Path("out_1.txt"), produces = Path("out_2.txt")): produces.write_text("2") """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -65,14 +61,12 @@ def task_2(produces): @pytest.mark.end_to_end() def test_two_tasks_have_the_same_product(tmp_path, runner, snapshot_cli): source = """ - import pytask + from pathlib import Path - @pytask.mark.produces("out.txt") - def task_1(produces): + def task_1(produces = Path("out.txt")): produces.write_text("1") - @pytask.mark.produces("out.txt") - def task_2(produces): + def task_2(produces = Path("out.txt")): produces.write_text("2") """ tmp_path.joinpath("task_d.py").write_text(textwrap.dedent(source)) @@ -89,10 +83,9 @@ def test_has_node_changed_catches_notnotfounderror(runner, tmp_path): """Missing nodes raise NodeNotFoundError when they do not exist and their state is requested.""" source = """ - import pytask + from pathlib import Path - @pytask.mark.produces("file.txt") - def task_example(produces): + def task_example(produces = Path("file.txt")): produces.write_text("test") """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) diff --git a/tests/test_dag_command.py b/tests/test_dag_command.py index 96143212..cf0113b7 100644 --- a/tests/test_dag_command.py +++ b/tests/test_dag_command.py @@ -35,10 +35,9 @@ def test_create_graph_via_cli(tmp_path, runner, format_, layout, rankdir): pytest.xfail("gvplugin_pango.dll might be missing on Github Actions.") source = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("input.txt") - def task_example(): pass + def task_example(path=Path("input.txt")): ... """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("input.txt").touch() @@ -77,8 +76,7 @@ def test_create_graph_via_task(tmp_path, runner, format_, layout, rankdir): from pathlib import Path import networkx as nx - @pytask.mark.depends_on("input.txt") - def task_example(depends_on): pass + def task_example(path=Path("input.txt")): ... def task_create_graph(): dag = pytask.build_dag({{"paths": Path(__file__).parent}}) @@ -106,10 +104,9 @@ def test_raise_error_with_graph_via_cli_missing_optional_dependency( monkeypatch, tmp_path, runner ): source = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("input.txt") - def task_example(): pass + def task_example(path=Path("input.txt")): ... """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("input.txt").touch() @@ -175,10 +172,9 @@ def test_raise_error_with_graph_via_cli_missing_optional_program( monkeypatch.setattr("_pytask.compat.shutil.which", lambda x: None) # noqa: ARG005 source = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("input.txt") - def task_example(): pass + def task_example(path=Path("input.txt")): ... """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("input.txt").touch() diff --git a/tests/test_database.py b/tests/test_database.py index 1639360c..82f6531d 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -17,11 +17,9 @@ def test_existence_of_hashes_in_db(tmp_path): """Modification dates of input and output files are stored in database.""" source = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("in.txt") - @pytask.mark.produces("out.txt") - def task_write(depends_on, produces): + def task_write(path=Path("in.txt"), produces=Path("out.txt")): produces.touch() """ task_path = tmp_path.joinpath("task_module.py") @@ -42,7 +40,7 @@ def task_write(depends_on, produces): with DatabaseSession() as db_session: task_id = session.tasks[0].signature out_path = tmp_path.joinpath("out.txt") - in_id = session.tasks[0].depends_on["depends_on"].signature + in_id = session.tasks[0].depends_on["path"].signature out_id = session.tasks[0].produces["produces"].signature for id_, path in ( diff --git a/tests/test_debugging.py b/tests/test_debugging.py index 77802a62..83e96743 100644 --- a/tests/test_debugging.py +++ b/tests/test_debugging.py @@ -71,12 +71,10 @@ def task_example(): @pytest.mark.skipif(sys.platform == "win32", reason="pexpect cannot spawn on Windows.") def test_post_mortem_on_error_w_kwargs(tmp_path): source = """ - import pytask from pathlib import Path - @pytask.mark.depends_on(Path(__file__).parent / "in.txt") - def task_example(depends_on): - a = depends_on.read_text() + def task_example(path=Path("in.txt")): + a = path.read_text() assert 0 """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -113,12 +111,10 @@ def task_example(): @pytest.mark.skipif(sys.platform == "win32", reason="pexpect cannot spawn on Windows.") def test_trace_w_kwargs(tmp_path): source = """ - import pytask from pathlib import Path - @pytask.mark.depends_on(Path(__file__).parent / "in.txt") - def task_example(depends_on): - print(depends_on.read_text()) + def task_example(path=Path("in.txt")): + print(path.read_text()) """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("I want you back.") diff --git a/tests/test_dry_run.py b/tests/test_dry_run.py index 13114c05..1c0e31dc 100644 --- a/tests/test_dry_run.py +++ b/tests/test_dry_run.py @@ -10,11 +10,9 @@ @pytest.mark.end_to_end() def test_dry_run(runner, tmp_path): source = """ - import pytask + from pathlib import Path - @pytask.mark.produces("out.txt") - def task_example(produces): - produces.touch() + def task_example(produces=Path("out.txt")): produces.touch() """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) @@ -29,20 +27,17 @@ def task_example(produces): def test_dry_run_w_subsequent_task(runner, tmp_path): """Subsequent tasks would be executed if their previous task changed.""" source = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("out.txt") - @pytask.mark.produces("out_2.txt") - def task_example(depends_on, produces): + def task_example(path=Path("out.txt"), produces=Path("out_2.txt")): produces.touch() """ tmp_path.joinpath("task_example_second.py").write_text(textwrap.dedent(source)) source = """ - import pytask + from pathlib import Path - @pytask.mark.produces("out.txt") - def task_example(produces): + def task_example(produces=Path("out.txt")): produces.touch() """ tmp_path.joinpath("task_example_first.py").write_text(textwrap.dedent(source)) @@ -67,20 +62,17 @@ def task_example(produces): def test_dry_run_w_subsequent_skipped_task(runner, tmp_path): """A skip is more important than a would be run.""" source_1 = """ - import pytask + from pathlib import Path - @pytask.mark.produces("out.txt") - def task_example(produces): + def task_example(produces=Path("out.txt")): produces.touch() """ tmp_path.joinpath("task_example_first.py").write_text(textwrap.dedent(source_1)) source_2 = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("out.txt") - @pytask.mark.produces("out_2.txt") - def task_example(depends_on, produces): + def task_example(path=Path("out.txt"), produces=Path("out_2.txt")): produces.touch() """ tmp_path.joinpath("task_example_second.py").write_text(textwrap.dedent(source_2)) @@ -106,12 +98,12 @@ def task_example(depends_on, produces): def test_dry_run_skip(runner, tmp_path): source = """ import pytask + from pathlib import Path @pytask.mark.skip def task_example_skip(): ... - @pytask.mark.produces("out.txt") - def task_example(produces): + def task_example(produces=Path("out.txt")): produces.touch() """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) @@ -128,14 +120,13 @@ def task_example(produces): def test_dry_run_skip_all(runner, tmp_path): source = """ import pytask + from pathlib import Path @pytask.mark.skip - @pytask.mark.produces("out.txt") - def task_example_skip(): ... + def task_example_skip(produces=Path("out.txt")): ... @pytask.mark.skip - @pytask.mark.depends_on("out.txt") - def task_example_skip_subsequent(): ... + def task_example_skip_subsequent(path=Path("out.txt")): ... """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) @@ -148,10 +139,9 @@ def task_example_skip_subsequent(): ... @pytest.mark.end_to_end() def test_dry_run_skipped_successful(runner, tmp_path): source = """ - import pytask + from pathlib import Path - @pytask.mark.produces("out.txt") - def task_example(produces): + def task_example(produces=Path("out.txt")): produces.touch() """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) @@ -171,10 +161,10 @@ def task_example(produces): def test_dry_run_persisted(runner, tmp_path): source = """ import pytask + from pathlib import Path @pytask.mark.persist - @pytask.mark.produces("out.txt") - def task_example(produces): + def task_example(produces=Path("out.txt")): produces.touch() """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) diff --git a/tests/test_execute.py b/tests/test_execute.py index ea8f0819..d2060bae 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -42,10 +42,9 @@ def test_execute_w_autocollect(runner, tmp_path): @pytest.mark.end_to_end() def test_task_did_not_produce_node(tmp_path): source = """ - import pytask + from pathlib import Path - @pytask.mark.produces("out.txt") - def task_example(): ... + def task_example(produces=Path("out.txt")): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -59,10 +58,9 @@ def task_example(): ... @pytest.mark.end_to_end() def test_task_did_not_produce_multiple_nodes_and_all_are_shown(runner, tmp_path): source = """ - import pytask + from pathlib import Path - @pytask.mark.produces(["1.txt", "2.txt"]) - def task_example(): ... + def task_example(produces=[Path("1.txt"), Path("2.txt")]): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -74,6 +72,21 @@ def task_example(): ... assert "2.txt" in result.output +@pytest.mark.end_to_end() +def test_missing_product(runner, tmp_path): + source = """ + from pathlib import Path + from typing_extensions import Annotated + from pytask import Product + + def task_with_non_path_dependency(path: Annotated[Path, Product]): ... + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.FAILED + + @pytest.mark.end_to_end() def test_node_not_found_in_task_setup(tmp_path): """Test for :class:`_pytask.exceptions.NodeNotFoundError` in task setup. @@ -112,46 +125,14 @@ def task_3(paths = [Path("deleted.txt"), Path("out_2.txt")]): assert isinstance(report.exc_info[1], NodeNotFoundError) -@pytest.mark.end_to_end() -@pytest.mark.parametrize( - "dependencies", - [[], ["in.txt"], ["in_1.txt", "in_2.txt"]], -) -@pytest.mark.parametrize("products", [["out.txt"], ["out_1.txt", "out_2.txt"]]) -def test_execution_w_varying_dependencies_products(tmp_path, dependencies, products): - source = f""" - import pytask - from pathlib import Path - - @pytask.mark.depends_on({dependencies}) - @pytask.mark.produces({products}) - def task_example(depends_on, produces): - if isinstance(produces, dict): - produces = produces.values() - elif isinstance(produces, Path): - produces = [produces] - for product in produces: - product.touch() - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - for dependency in dependencies: - tmp_path.joinpath(dependency).touch() - - session = build(paths=tmp_path) - assert session.exit_code == ExitCode.OK - - @pytest.mark.end_to_end() def test_depends_on_and_produces_can_be_used_in_task(tmp_path): source = """ - import pytask from pathlib import Path - @pytask.mark.depends_on("in.txt") - @pytask.mark.produces("out.txt") - def task_example(depends_on, produces): - assert isinstance(depends_on, Path) and isinstance(produces, Path) - produces.write_text(depends_on.read_text()) + def task_example(path=Path("in.txt"), produces=Path("out.txt")): + assert isinstance(path, Path) and isinstance(produces, Path) + produces.write_text(path.read_text()) """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("Here I am. Once again.") @@ -162,90 +143,6 @@ def task_example(depends_on, produces): assert tmp_path.joinpath("out.txt").read_text() == "Here I am. Once again." -@pytest.mark.end_to_end() -def test_assert_multiple_dependencies_are_merged_to_dict(tmp_path, runner): - source = """ - import pytask - from pathlib import Path - - @pytask.mark.depends_on({3: "in_3.txt", 4: "in_4.txt"}) - @pytask.mark.depends_on(["in_1.txt", "in_2.txt"]) - @pytask.mark.depends_on("in_0.txt") - @pytask.mark.produces("out.txt") - def task_example(depends_on, produces): - expected = { - i: Path(__file__).parent.joinpath(f"in_{i}.txt").resolve() - for i in range(5) - } - assert depends_on == expected - produces.touch() - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - for name in [f"in_{i}.txt" for i in range(5)]: - tmp_path.joinpath(name).touch() - - result = runner.invoke(cli, [tmp_path.as_posix()]) - - assert result.exit_code == ExitCode.OK - - -@pytest.mark.end_to_end() -def test_assert_multiple_products_are_merged_to_dict(tmp_path, runner): - source = """ - import pytask - from pathlib import Path - - @pytask.mark.depends_on("in.txt") - @pytask.mark.produces({3: "out_3.txt", 4: "out_4.txt"}) - @pytask.mark.produces(["out_1.txt", "out_2.txt"]) - @pytask.mark.produces("out_0.txt") - def task_example(depends_on, produces): - expected = { - i: Path(__file__).parent.joinpath(f"out_{i}.txt").resolve() - for i in range(5) - } - assert produces == expected - for product in produces.values(): - product.touch() - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - tmp_path.joinpath("in.txt").touch() - - result = runner.invoke(cli, [tmp_path.as_posix()]) - - assert result.exit_code == ExitCode.OK - - -@pytest.mark.end_to_end() -@pytest.mark.parametrize("input_type", ["list", "dict"]) -def test_preserve_input_for_dependencies_and_products(tmp_path, input_type): - """Input type for dependencies and products is preserved.""" - path = tmp_path.joinpath("in.txt") - input_ = {0: path.as_posix()} if input_type == "dict" else [path.as_posix()] - path.touch() - - path = tmp_path.joinpath("out.txt") - output = {0: path.as_posix()} if input_type == "dict" else [path.as_posix()] - - source = f""" - import pytask - from pathlib import Path - - @pytask.mark.depends_on({input_}) - @pytask.mark.produces({output}) - def task_example(depends_on, produces): - for nodes in [depends_on, produces]: - assert isinstance(nodes, dict) - assert len(nodes) == 1 - assert 0 in nodes - produces[0].touch() - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - session = build(paths=tmp_path) - assert session.exit_code == ExitCode.OK - - @pytest.mark.end_to_end() @pytest.mark.parametrize("n_failures", [1, 2, 3]) def test_execution_stops_after_n_failures(tmp_path, n_failures): @@ -329,13 +226,11 @@ def task_error(): raise ValueError @pytest.mark.parametrize("verbose", [1, 2]) def test_traceback_of_previous_task_failed_is_not_shown(runner, tmp_path, verbose): source = """ - import pytask + from pathlib import Path - @pytask.mark.produces("in.txt") - def task_first(): raise ValueError + def task_first(produces=Path("in.txt")): raise ValueError - @pytask.mark.depends_on("in.txt") - def task_second(): pass + def task_second(path=Path("in.txt")): ... """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) @@ -429,9 +324,8 @@ def task_example({arg_name}: Annotated[Path, Product] = Path("out.txt")) -> None def test_task_errors_with_nested_product_annotation(tmp_path): source = """ from pathlib import Path - from typing_extensions import Annotated + from typing_extensions import Annotated, Dict from pytask import Product - from typing import Dict def task_example( paths_to_file: Dict[str, Annotated[Path, Product]] = {"a": Path("out.txt")} @@ -461,8 +355,7 @@ def test_task_with_hashed_python_node(runner, tmp_path, definition): import json from pathlib import Path from pytask import Product, PythonNode - from typing import Any - from typing_extensions import Annotated + from typing_extensions import Annotated, Any data = json.loads(Path(__file__).parent.joinpath("data.json").read_text()) @@ -711,9 +604,7 @@ def test_execute_tasks_via_functional_api(tmp_path): import sys from pathlib import Path from typing_extensions import Annotated - from pytask import PathNode - import pytask - from pytask import PythonNode + from pytask import PathNode, PythonNode, build node_text = PythonNode() @@ -726,7 +617,7 @@ def create_file(content: Annotated[str, node_text]) -> Annotated[str, node_file] return content if __name__ == "__main__": - session = pytask.build(tasks=[create_file, create_text]) + session = build(tasks=[create_file, create_text]) assert len(session.tasks) == 2 assert len(session.dag.nodes) == 5 sys.exit(session.exit_code) @@ -748,7 +639,6 @@ def test_pass_non_task_to_functional_api_that_are_ignored(): @pytest.mark.end_to_end() def test_multiple_product_annotations(runner, tmp_path): source = """ - from __future__ import annotations from pytask import Product from typing_extensions import Annotated from pathlib import Path @@ -879,11 +769,9 @@ def task_write_file(text: Annotated[str, node]) -> Annotated[str, Path("file.txt @pytest.mark.end_to_end() def test_check_if_root_nodes_are_available(tmp_path, runner): source = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("in.txt") - @pytask.mark.produces("out.txt") - def task_d(produces): + def task_d(path=Path("in.txt"), produces=Path("out.txt")): produces.write_text("1") """ tmp_path.joinpath("task_d.py").write_text(textwrap.dedent(source)) @@ -920,11 +808,9 @@ def test_check_if_root_nodes_are_available_with_separate_build_folder(tmp_path, tmp_path.joinpath("src").mkdir() tmp_path.joinpath("bld").mkdir() source = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("../bld/in.txt") - @pytask.mark.produces("out.txt") - def task_d(produces): + def task_d(path=Path("../bld/in.txt"), produces=Path("out.txt")): produces.write_text("1") """ tmp_path.joinpath("src", "task_d.py").write_text(textwrap.dedent(source)) diff --git a/tests/test_live.py b/tests/test_live.py index 468ce36b..65760fc5 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -259,12 +259,13 @@ def test_live_execution_skips_do_not_crowd_out_displayed_tasks(capsys, tmp_path) @pytest.mark.end_to_end() def test_full_execution_table_is_displayed_at_the_end_of_execution(tmp_path, runner): source = """ - import pytask + from pytask import task + from pathlib import Path - for produces in [f"{i}.txt" for i in range(4)]: + for i in range(4): - @pytask.mark.task - def task_create_file(produces=produces): + @task + def task_create_file(produces=Path(f"{i}.txt")): produces.touch() """ # Subfolder to reduce task id and be able to check the output later. @@ -277,7 +278,7 @@ def task_create_file(produces=produces): assert result.exit_code == ExitCode.OK for i in range(4): - assert f"{i}.txt" in result.output + assert f"[produces{i}]" in result.output @pytest.mark.end_to_end() diff --git a/tests/test_mark.py b/tests/test_mark.py index 8f27404a..74eb916c 100644 --- a/tests/test_mark.py +++ b/tests/test_mark.py @@ -1,6 +1,5 @@ from __future__ import annotations -import subprocess import sys import textwrap @@ -253,16 +252,12 @@ def test_configuration_failed(runner, tmp_path): @pytest.mark.end_to_end() def test_selecting_task_with_keyword_should_run_predecessor(runner, tmp_path): source = """ - import pytask + from pathlib import Path - @pytask.mark.produces("first.txt") - def task_first(produces): + def task_first(produces=Path("first.txt")): produces.touch() - - @pytask.mark.depends_on("first.txt") - def task_second(depends_on): - pass + def task_second(path=Path("first.txt")): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -276,15 +271,13 @@ def task_second(depends_on): def test_selecting_task_with_marker_should_run_predecessor(runner, tmp_path): source = """ import pytask + from pathlib import Path - @pytask.mark.produces("first.txt") - def task_first(produces): + def task_first(produces=Path("first.txt")): produces.touch() @pytask.mark.wip - @pytask.mark.depends_on("first.txt") - def task_second(depends_on): - pass + def task_second(path=Path("first.txt")): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -298,14 +291,11 @@ def task_second(depends_on): @pytest.mark.end_to_end() def test_selecting_task_with_keyword_ignores_other_task(runner, tmp_path): source = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("first.txt") - def task_first(): - pass + def task_first(path=Path("first.txt")): ... - def task_second(): - pass + def task_second(): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -320,14 +310,12 @@ def task_second(): def test_selecting_task_with_marker_ignores_other_task(runner, tmp_path): source = """ import pytask + from pathlib import Path - @pytask.mark.depends_on("first.txt") - def task_first(): - pass + def task_first(path=Path("first.txt")): ... @pytask.mark.wip - def task_second(): - pass + def task_second(): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -345,8 +333,7 @@ def test_selecting_task_with_unknown_marker_raises_warning(runner, tmp_path): import pytask @pytask.mark.wip - def task_example(): - pass + def task_example(): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -357,46 +344,6 @@ def task_example(): assert "Warnings" in result.output -@pytest.mark.end_to_end() -def test_deprecation_warnings_for_decorators(tmp_path): - source = """ - import pytask - - @pytask.mark.depends_on("in.txt") - @pytask.mark.produces("out.txt") - def task_write_text(depends_on, produces): - ... - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - tmp_path.joinpath("in.txt").touch() - - result = subprocess.run( - ("pytest", tmp_path.joinpath("task_module.py").as_posix()), - capture_output=True, - check=False, - ) - assert b"FutureWarning: '@pytask.mark.depends_on'" in result.stdout - assert b"FutureWarning: '@pytask.mark.produces'" in result.stdout - - -@pytest.mark.end_to_end() -def test_deprecation_warnings_for_task_decorator(tmp_path): - source = """ - import pytask - - @pytask.mark.task - def task_write_text(): ... - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - result = subprocess.run( - ("pytest", tmp_path.joinpath("task_module.py").as_posix()), - capture_output=True, - check=False, - ) - assert b"FutureWarning: '@pytask.mark.task'" in result.stdout - - @pytest.mark.end_to_end() def test_different_mark_import(runner, tmp_path): source = """ @@ -426,7 +373,22 @@ def task_write_text(): ... @pytest.mark.end_to_end() -def test_error_with_parametrize(runner, tmp_path): +@pytest.mark.parametrize("name", ["parametrize", "depends_on", "produces"]) +def test_error_with_depreacated_markers(runner, tmp_path, name): + source = f""" + from pytask import mark + + @mark.{name} + def task_write_text(): ... + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.COLLECTION_FAILED + assert f"@pytask.mark.{name}" in result.output + + +@pytest.mark.end_to_end() +def test_error_with_d(runner, tmp_path): source = """ from pytask import mark diff --git a/tests/test_mark_cli.py b/tests/test_mark_cli.py index 23c0f54b..04949a5e 100644 --- a/tests/test_mark_cli.py +++ b/tests/test_mark_cli.py @@ -15,11 +15,14 @@ def test_show_markers(runner): assert all( marker in result.output for marker in ( - "depends_on", - "produces", + "filterwarnings", + "persist", "skip", "skip_ancestor_failed", "skip_unchanged", + "skipif", + "try_first", + "try_last", ) ) diff --git a/tests/test_mark_utils.py b/tests/test_mark_utils.py index fdcbc2a2..f0d82d31 100644 --- a/tests/test_mark_utils.py +++ b/tests/test_mark_utils.py @@ -19,12 +19,12 @@ [ ([], []), ( - [pytask.mark.produces(), pytask.mark.depends_on()], - [pytask.mark.produces(), pytask.mark.depends_on()], + [pytask.mark.mark1(), pytask.mark.mark2()], + [pytask.mark.mark1(), pytask.mark.mark2()], ), ( - [pytask.mark.produces(), pytask.mark.produces(), pytask.mark.depends_on()], - [pytask.mark.produces(), pytask.mark.produces(), pytask.mark.depends_on()], + [pytask.mark.mark1(), pytask.mark.mark1(), pytask.mark.mark2()], + [pytask.mark.mark1(), pytask.mark.mark1(), pytask.mark.mark2()], ), ], ) @@ -41,12 +41,12 @@ def test_get_all_marks_from_task(markers, expected): (None, []), ([], []), ( - [pytask.mark.produces(), pytask.mark.depends_on()], - [pytask.mark.produces(), pytask.mark.depends_on()], + [pytask.mark.mark1(), pytask.mark.mark2()], + [pytask.mark.mark1(), pytask.mark.mark2()], ), ( - [pytask.mark.produces(), pytask.mark.produces(), pytask.mark.depends_on()], - [pytask.mark.produces(), pytask.mark.produces(), pytask.mark.depends_on()], + [pytask.mark.mark1(), pytask.mark.mark1(), pytask.mark.mark2()], + [pytask.mark.mark1(), pytask.mark.mark1(), pytask.mark.mark2()], ), ], ) @@ -67,14 +67,14 @@ def func(): [ ([], "not_found", []), ( - [pytask.mark.produces(), pytask.mark.depends_on()], - "produces", - [pytask.mark.produces()], + [pytask.mark.mark1(), pytask.mark.mark2()], + "mark1", + [pytask.mark.mark1()], ), ( - [pytask.mark.produces(), pytask.mark.produces(), pytask.mark.depends_on()], - "produces", - [pytask.mark.produces(), pytask.mark.produces()], + [pytask.mark.mark1(), pytask.mark.mark1(), pytask.mark.mark2()], + "mark1", + [pytask.mark.mark1(), pytask.mark.mark1()], ), ], ) @@ -91,14 +91,14 @@ def test_get_marks_from_task(markers, marker_name, expected): (None, "not_found", []), ([], "not_found", []), ( - [pytask.mark.produces(), pytask.mark.depends_on()], - "produces", - [pytask.mark.produces()], + [pytask.mark.mark1(), pytask.mark.mark2()], + "mark1", + [pytask.mark.mark1()], ), ( - [pytask.mark.produces(), pytask.mark.produces(), pytask.mark.depends_on()], - "produces", - [pytask.mark.produces(), pytask.mark.produces()], + [pytask.mark.mark1(), pytask.mark.mark1(), pytask.mark.mark2()], + "mark1", + [pytask.mark.mark1(), pytask.mark.mark1()], ), ], ) @@ -117,14 +117,14 @@ def func(): @pytest.mark.parametrize( ("markers", "marker_name", "expected"), [ - ([pytask.mark.produces()], "not_found", False), + ([pytask.mark.mark1()], "not_found", False), ( - [pytask.mark.produces(), pytask.mark.depends_on()], - "produces", + [pytask.mark.mark1(), pytask.mark.mark2()], + "mark1", True, ), ( - [pytask.mark.produces(), pytask.mark.produces(), pytask.mark.depends_on()], + [pytask.mark.mark1(), pytask.mark.mark1(), pytask.mark.mark2()], "other", False, ), @@ -142,10 +142,10 @@ def test_has_mark_for_task(markers, marker_name, expected): [ (None, "not_found", False), ([], "not_found", False), - ([pytask.mark.produces(), pytask.mark.depends_on()], "produces", True), + ([pytask.mark.mark1(), pytask.mark.mark2()], "mark1", True), ( - [pytask.mark.produces(), pytask.mark.produces(), pytask.mark.depends_on()], - "produces", + [pytask.mark.mark1(), pytask.mark.mark1(), pytask.mark.mark2()], + "mark1", True, ), ], @@ -167,16 +167,16 @@ def func(): [ ([], "not_found", [], []), ( - [pytask.mark.produces(), pytask.mark.depends_on()], - "produces", - [pytask.mark.produces()], - [pytask.mark.depends_on()], + [pytask.mark.mark1(), pytask.mark.mark2()], + "mark1", + [pytask.mark.mark1()], + [pytask.mark.mark2()], ), ( - [pytask.mark.produces(), pytask.mark.produces(), pytask.mark.depends_on()], - "produces", - [pytask.mark.produces(), pytask.mark.produces()], - [pytask.mark.depends_on()], + [pytask.mark.mark1(), pytask.mark.mark1(), pytask.mark.mark2()], + "mark1", + [pytask.mark.mark1(), pytask.mark.mark1()], + [pytask.mark.mark2()], ), ], ) @@ -196,16 +196,16 @@ def test_remove_marks_from_task( (None, "not_found", [], []), ([], "not_found", [], []), ( - [pytask.mark.produces(), pytask.mark.depends_on()], - "produces", - [pytask.mark.produces()], - [pytask.mark.depends_on()], + [pytask.mark.mark1(), pytask.mark.mark2()], + "mark1", + [pytask.mark.mark1()], + [pytask.mark.mark2()], ), ( - [pytask.mark.produces(), pytask.mark.produces(), pytask.mark.depends_on()], - "produces", - [pytask.mark.produces(), pytask.mark.produces()], - [pytask.mark.depends_on()], + [pytask.mark.mark1(), pytask.mark.mark1(), pytask.mark.mark2()], + "mark1", + [pytask.mark.mark1(), pytask.mark.mark1()], + [pytask.mark.mark2()], ), ], ) @@ -229,8 +229,8 @@ def func(): "markers", [ [], - [pytask.mark.produces(), pytask.mark.depends_on()], - [pytask.mark.produces(), pytask.mark.produces(), pytask.mark.depends_on()], + [pytask.mark.mark1(), pytask.mark.mark2()], + [pytask.mark.mark1(), pytask.mark.mark1(), pytask.mark.mark2()], ], ) def test_set_marks_to_task(markers): @@ -244,8 +244,8 @@ def test_set_marks_to_task(markers): "markers", [ [], - [pytask.mark.produces(), pytask.mark.depends_on()], - [pytask.mark.produces(), pytask.mark.produces(), pytask.mark.depends_on()], + [pytask.mark.mark1(), pytask.mark.mark2()], + [pytask.mark.mark1(), pytask.mark.mark1(), pytask.mark.mark2()], ], ) def test_set_marks_to_obj(markers): diff --git a/tests/test_persist.py b/tests/test_persist.py index db57e2c8..8909b405 100644 --- a/tests/test_persist.py +++ b/tests/test_persist.py @@ -39,12 +39,11 @@ def test_multiple_runs_with_persist(tmp_path): """ source = """ import pytask + from pathlib import Path @pytask.mark.persist - @pytask.mark.depends_on("in.txt") - @pytask.mark.produces("out.txt") - def task_dummy(depends_on, produces): - produces.write_text(depends_on.read_text()) + def task_dummy(path=Path("in.txt"), produces=Path("out.txt")): + produces.write_text(path.read_text()) """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("I'm not the reason you care.") @@ -91,11 +90,10 @@ def task_dummy(depends_on, produces): def test_migrating_a_whole_task_with_persist(tmp_path): source = """ import pytask + from pathlib import Path @pytask.mark.persist - @pytask.mark.depends_on("in.txt") - @pytask.mark.produces("out.txt") - def task_dummy(depends_on, produces): + def task_dummy(depends_on=Path("in.txt"), produces=Path("out.txt")): produces.write_text(depends_on.read_text()) """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) diff --git a/tests/test_profile.py b/tests/test_profile.py index 6c3bfcda..2bd8c031 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -63,10 +63,9 @@ def task_example(): time.sleep(2) def test_profile_if_there_is_information_on_collected_tasks(tmp_path, runner): source = """ import time - import pytask + from pathlib import Path - @pytask.mark.produces("out.txt") - def task_example(produces): + def task_example(produces=Path("out.txt")): time.sleep(2) produces.write_text("There are nine billion bicycles in Beijing.") """ diff --git a/tests/test_skipping.py b/tests/test_skipping.py index 177c3a4c..fbe0606d 100644 --- a/tests/test_skipping.py +++ b/tests/test_skipping.py @@ -40,12 +40,10 @@ def task_dummy(): @pytest.mark.end_to_end() def test_skip_unchanged_w_dependencies_and_products(tmp_path): source = """ - import pytask + from pathlib import Path - @pytask.mark.depends_on("in.txt") - @pytask.mark.produces("out.txt") - def task_dummy(depends_on, produces): - produces.write_text(depends_on.read_text()) + def task_dummy(path=Path("in.txt"), produces=Path("out.txt")): + produces.write_text(path.read_text()) """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("Original content of in.txt.") @@ -65,15 +63,12 @@ def task_dummy(depends_on, produces): @pytest.mark.end_to_end() def test_skipif_ancestor_failed(tmp_path): source = """ - import pytask + from pathlib import Path - @pytask.mark.produces("out.txt") - def task_first(): + def task_first(produces=Path("out.txt")): assert 0 - @pytask.mark.depends_on("out.txt") - def task_second(): - pass + def task_second(path=Path("out.txt")): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -89,15 +84,13 @@ def task_second(): def test_if_skip_decorator_is_applied_to_following_tasks(tmp_path): source = """ import pytask + from pathlib import Path @pytask.mark.skip - @pytask.mark.produces("out.txt") - def task_first(): + def task_first(produces=Path("out.txt")): assert 0 - @pytask.mark.depends_on("out.txt") - def task_second(): - pass + def task_second(path=Path("out.txt")): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -116,10 +109,10 @@ def task_second(): def test_skip_if_dependency_is_missing(tmp_path, mark_string): source = f""" import pytask + from pathlib import Path {mark_string} - @pytask.mark.depends_on("in.txt") - def task_first(): + def task_first(path=Path("in.txt")): assert 0 """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -137,14 +130,13 @@ def task_first(): def test_skip_if_dependency_is_missing_only_for_one_task(runner, tmp_path, mark_string): source = f""" import pytask + from pathlib import Path {mark_string} - @pytask.mark.depends_on("in.txt") - def task_first(): + def task_first(path=Path("in.txt")): assert 0 - @pytask.mark.depends_on("in.txt") - def task_second(): + def task_second(path=Path("in.txt")): assert 0 """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -161,14 +153,13 @@ def task_second(): def test_if_skipif_decorator_is_applied_skipping(tmp_path): source = """ import pytask + from pathlib import Path @pytask.mark.skipif(condition=True, reason="bla") - @pytask.mark.produces("out.txt") - def task_first(): + def task_first(produces=Path("out.txt")): assert False - @pytask.mark.depends_on("out.txt") - def task_second(): + def task_second(path=Path("out.txt")): assert False """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -191,15 +182,13 @@ def task_second(): def test_if_skipif_decorator_is_applied_execute(tmp_path): source = """ import pytask + from pathlib import Path @pytask.mark.skipif(False, reason="bla") - @pytask.mark.produces("out.txt") - def task_first(produces): + def task_first(produces=Path("out.txt")): produces.touch() - @pytask.mark.depends_on("out.txt") - def task_second(depends_on): - pass + def task_second(path=Path("out.txt")): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -221,15 +210,14 @@ def test_if_skipif_decorator_is_applied_any_condition_matches(tmp_path): """Any condition of skipif has to be True and only their message is shown.""" source = """ import pytask + from pathlib import Path @pytask.mark.skipif(condition=False, reason="I am fine") @pytask.mark.skipif(condition=True, reason="No, I am not.") - @pytask.mark.produces("out.txt") - def task_first(): + def task_first(produces=Path("out.txt")): assert False - @pytask.mark.depends_on("out.txt") - def task_second(): + def task_second(path=Path("out.txt")): assert False """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) diff --git a/tests/test_task.py b/tests/test_task.py index e4b00014..968480e8 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -14,11 +14,11 @@ def test_task_with_task_decorator(tmp_path, func_name, task_name): task_decorator_input = f"{task_name!r}" if task_name else task_name source = f""" - import pytask + from pytask import task + from pathlib import Path - @pytask.mark.task({task_decorator_input}) - @pytask.mark.produces("out.txt") - def {func_name}(produces): + @task({task_decorator_input}) + def {func_name}(produces=Path("out.txt")): produces.write_text("Hello. It's me.") """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -36,13 +36,13 @@ def {func_name}(produces): @pytest.mark.end_to_end() def test_parametrization_in_for_loop(tmp_path, runner): source = """ - import pytask + from pytask import task + from pathlib import Path for i in range(2): - @pytask.mark.task - @pytask.mark.produces(f"out_{i}.txt") - def task_example(produces): + @task + def task_example(produces=Path(f"out_{i}.txt")): produces.write_text("Your advertisement could be here.") """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -57,15 +57,14 @@ def task_example(produces): @pytest.mark.end_to_end() def test_parametrization_in_for_loop_from_markers(tmp_path, runner): source = """ - import pytask + from pytask import task + from pathlib import Path for i in range(2): - @pytask.mark.task - @pytask.mark.depends_on(f"in_{i}.txt") - @pytask.mark.produces(f"out_{i}.txt") - def example(depends_on, produces): - produces.write_text(depends_on.read_text()) + @task + def example(path=Path(f"in_{i}.txt"), produces=Path(f"out_{i}.txt")): + produces.write_text(path.read_text()) """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in_0.txt").write_text("Your advertisement could be here.") @@ -74,20 +73,21 @@ def example(depends_on, produces): result = runner.invoke(cli, [tmp_path.as_posix()]) assert result.exit_code == ExitCode.OK - assert "example[depends_on0-produces0]" in result.output - assert "example[depends_on1-produces1]" in result.output + assert "example[path0-produces0]" in result.output + assert "example[path1-produces1]" in result.output @pytest.mark.end_to_end() def test_parametrization_in_for_loop_from_signature(tmp_path, runner): source = """ - import pytask + from pytask import task + from pathlib import Path for i in range(2): - @pytask.mark.task - def example(depends_on=f"in_{i}.txt", produces=f"out_{i}.txt"): - produces.write_text(depends_on.read_text()) + @task + def example(path=Path(f"in_{i}.txt"), produces=Path(f"out_{i}.txt")): + produces.write_text(path.read_text()) """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in_0.txt").write_text("Your advertisement could be here.") @@ -96,20 +96,20 @@ def example(depends_on=f"in_{i}.txt", produces=f"out_{i}.txt"): result = runner.invoke(cli, [tmp_path.as_posix()]) assert result.exit_code == ExitCode.OK - assert "example[in_0.txt-out_0.txt]" in result.output - assert "example[in_1.txt-out_1.txt]" in result.output + assert "example[path0-produces0]" in result.output + assert "example[path1-produces1]" in result.output @pytest.mark.end_to_end() def test_parametrization_in_for_loop_from_markers_and_args(tmp_path, runner): source = """ - import pytask + from pytask import task + from pathlib import Path for i in range(2): - @pytask.mark.task - @pytask.mark.produces(f"out_{i}.txt") - def example(produces, i=i): + @task + def example(produces=Path(f"out_{i}.txt"), i=i): produces.write_text(str(i)) """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -124,11 +124,12 @@ def example(produces, i=i): @pytest.mark.end_to_end() def test_parametrization_in_for_loop_from_decorator(tmp_path, runner): source = """ - import pytask + from pytask import task + from pathlib import Path for i in range(2): - @pytask.mark.task(name="deco_task", kwargs={"i": i, "produces": f"out_{i}.txt"}) + @task(name="deco_task", kwargs={"i": i, "produces": Path(f"out_{i}.txt")}) def example(produces, i): produces.write_text(str(i)) """ @@ -137,20 +138,19 @@ def example(produces, i): result = runner.invoke(cli, [tmp_path.as_posix()]) assert result.exit_code == ExitCode.OK - assert "deco_task[out_0.txt-0]" in result.output - assert "deco_task[out_1.txt-1]" in result.output + assert "deco_task[produces0-0]" in result.output + assert "deco_task[produces1-1]" in result.output @pytest.mark.end_to_end() def test_parametrization_in_for_loop_with_ids(tmp_path, runner): source = """ - import pytask + from pytask import task + from pathlib import Path for i in range(2): - @pytask.mark.task( - "deco_task", id=str(i), kwargs={"i": i, "produces": f"out_{i}.txt"} - ) + @task("deco_task", id=str(i), kwargs={"i": i, "produces": Path(f"out_{i}.txt")}) def example(produces, i): produces.write_text(str(i)) """ @@ -166,12 +166,13 @@ def example(produces, i): @pytest.mark.end_to_end() def test_parametrization_in_for_loop_with_error(tmp_path, runner): source = """ - import pytask + from pytask import task + from pathlib import Path for i in range(2): - @pytask.mark.task - def task_example(produces=f"out_{i}.txt"): + @task + def task_example(produces=Path(f"out_{i}.txt")): raise ValueError """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -181,23 +182,24 @@ def task_example(produces=f"out_{i}.txt"): assert result.exit_code == ExitCode.FAILED assert "2 Failed" in result.output assert "Traceback" in result.output - assert "task_example[out_0.txt]" in result.output - assert "task_example[out_1.txt]" in result.output + assert "task_example[produces0]" in result.output + assert "task_example[produces1]" in result.output @pytest.mark.end_to_end() def test_parametrization_in_for_loop_from_decorator_w_irregular_dicts(tmp_path, runner): source = """ - import pytask + from pytask import task + from pathlib import Path ID_TO_KWARGS = { - "first": {"i": 0, "produces": "out_0.txt"}, - "second": {"produces": "out_1.txt"}, + "first": {"i": 0, "produces": Path("out_0.txt")}, + "second": {"produces": Path("out_1.txt")}, } for id_, kwargs in ID_TO_KWARGS.items(): - @pytask.mark.task(name="deco_task", id=id_, kwargs=kwargs) + @task(name="deco_task", id=id_, kwargs=kwargs) def example(produces, i): produces.write_text(str(i)) """ @@ -216,13 +218,13 @@ def example(produces, i): @pytest.mark.end_to_end() def test_parametrization_in_for_loop_with_one_iteration(tmp_path, runner): source = """ - import pytask + from pytask import task + from pathlib import Path for i in range(1): - @pytask.mark.task - @pytask.mark.produces(f"out_{i}.txt") - def task_example(produces): + @task + def task_example(produces=Path(f"out_{i}.txt")): produces.write_text("Your advertisement could be here.") """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -237,18 +239,17 @@ def task_example(produces): @pytest.mark.end_to_end() def test_parametrization_in_for_loop_and_normal(tmp_path, runner): source = """ - import pytask + from pytask import task + from pathlib import Path for i in range(1): - @pytask.mark.task - @pytask.mark.produces(f"out_{i}.txt") - def task_example(produces): + @task + def task_example(produces=Path(f"out_{i}.txt")): produces.write_text("Your advertisement could be here.") - @pytask.mark.task - @pytask.mark.produces(f"out_1.txt") - def task_example(produces): + @task + def task_example(produces=Path(f"out_1.txt")): produces.write_text("Your advertisement could be here.") """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -264,18 +265,17 @@ def task_example(produces): @pytest.mark.end_to_end() def test_parametrized_names_without_parametrization(tmp_path, runner): source = """ - import pytask + from pytask import task + from pathlib import Path for i in range(2): - @pytask.mark.task - @pytask.mark.produces(f"out_{i}.txt") - def task_example(produces): + @task + def task_example(produces=Path(f"out_{i}.txt")): produces.write_text("Your advertisement could be here.") - @pytask.mark.task - @pytask.mark.produces("out_2.txt") - def task_example(produces): + @task + def task_example(produces=Path("out_2.txt")): produces.write_text("Your advertisement could be here.") """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -292,12 +292,12 @@ def task_example(produces): @pytest.mark.end_to_end() def test_order_of_decorator_does_not_matter(tmp_path, runner): source = """ - import pytask + from pytask import task, mark + from pathlib import Path - @pytask.mark.skip - @pytask.mark.task - @pytask.mark.produces(f"out.txt") - def task_example(produces): + @task + @mark.skip + def task_example(produces=Path(f"out.txt")): produces.write_text("Your advertisement could be here.") """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -311,15 +311,13 @@ def task_example(produces): @pytest.mark.end_to_end() def test_task_function_with_partialed_args(tmp_path, runner): source = """ - import pytask import functools + from pathlib import Path def func(produces, content): produces.write_text(content) - task_func = pytask.mark.produces("out.txt")( - functools.partial(func, content="hello") - ) + task_func = functools.partial(func, content="hello", produces=Path("out.txt")) """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -363,29 +361,26 @@ def test_parametrized_tasks_without_arguments_in_signature(tmp_path, runner): """ source = f""" - import pytask + from pytask import task from pathlib import Path for i in range(1): - @pytask.mark.task - @pytask.mark.produces(f"out_{{i}}.txt") - def task_example(): + @task + def task_example(produces=Path(f"out_{{i}}.txt")): Path("{tmp_path.as_posix()}").joinpath(f"out_{{i}}.txt").write_text( "I use globals. How funny." ) - @pytask.mark.task - @pytask.mark.produces("out_1.txt") - def task_example(): + @task + def task_example(produces=Path("out_1.txt")): Path("{tmp_path.as_posix()}").joinpath("out_1.txt").write_text( "I use globals. How funny." ) - @pytask.mark.task(id="hello") - @pytask.mark.produces("out_2.txt") - def task_example(): + @task(id="hello") + def task_example(produces=Path("out_2.txt")): Path("{tmp_path.as_posix()}").joinpath("out_2.txt").write_text( "I use globals. How funny." ) @@ -395,8 +390,8 @@ def task_example(): result = runner.invoke(cli, [tmp_path.as_posix()]) assert result.exit_code == ExitCode.OK - assert "task_example[0]" in result.output - assert "task_example[1]" in result.output + assert "task_example[produces0]" in result.output + assert "task_example[produces1]" in result.output assert "task_example[hello]" in result.output assert "Collected 3 tasks" in result.output @@ -404,10 +399,10 @@ def task_example(): @pytest.mark.end_to_end() def test_that_dynamically_creates_tasks_are_captured(runner, tmp_path): source = """ - import pytask + from pytask import task _DEFINITION = ''' - @pytask.mark.task + @task def task_example(): pass ''' @@ -431,9 +426,9 @@ def task_example(): ) def test_raise_errors_for_irregular_ids(runner, tmp_path, irregular_id): source = f""" - import pytask + from pytask import task - @pytask.mark.task(id={irregular_id}) + @task(id={irregular_id}) def task_example(): pass """ @@ -465,9 +460,9 @@ def task_func(i=i): @pytest.mark.end_to_end() def test_task_receives_unknown_kwarg(runner, tmp_path): source = """ - import pytask + from pytask import task - @pytask.mark.task(kwargs={"i": 1}) + @task(kwargs={"i": 1}) def task_example(): pass """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -478,20 +473,18 @@ def task_example(): pass @pytest.mark.end_to_end() def test_task_receives_namedtuple(runner, tmp_path): source = """ - import pytask from typing_extensions import NamedTuple, Annotated from pathlib import Path - from pytask import Product, PythonNode + from pytask import Product, PythonNode, task class Args(NamedTuple): path_in: Path arg: str path_out: Path - args = Args(Path("input.txt"), "world!", Path("output.txt")) - @pytask.mark.task(kwargs=args) + @task(kwargs=args) def task_example( path_in: Path, arg: str, path_out: Annotated[Path, Product] ) -> None: @@ -508,12 +501,11 @@ def task_example( @pytest.mark.end_to_end() def test_task_kwargs_overwrite_default_arguments(runner, tmp_path): source = """ - import pytask - from pytask import Product + from pytask import Product, task from pathlib import Path from typing_extensions import Annotated - @pytask.mark.task(kwargs={ + @task(kwargs={ "in_path": Path("in.txt"), "addition": "world!", "out_path": Path("out.txt") }) def task_example( diff --git a/tests/test_tree_util.py b/tests/test_tree_util.py index 8c93cb15..7b9be7a9 100644 --- a/tests/test_tree_util.py +++ b/tests/test_tree_util.py @@ -12,22 +12,19 @@ @pytest.mark.end_to_end() -@pytest.mark.parametrize("decorator_name", ["depends_on", "produces"]) -def test_task_with_complex_product_did_not_produce_node(tmp_path, decorator_name): +@pytest.mark.parametrize("arg_name", ["depends_on", "produces"]) +def test_task_with_complex_product_did_not_produce_node(tmp_path, arg_name): source = f""" - import pytask - + from pathlib import Path complex = [ - "out.txt", - ("tuple_out.txt",), - ["list_out.txt"], - {{"a": "dict_out.txt", "b": {{"c": "dict_out_2.txt"}}}}, + Path("out.txt"), + (Path("tuple_out.txt"),), + [Path("list_out.txt")], + {{"a": Path("dict_out.txt"), "b": {{"c": Path("dict_out_2.txt")}}}}, ] - - @pytask.mark.{decorator_name}(complex) - def task_example(): + def task_example({arg_name}=complex): pass """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -36,14 +33,14 @@ def task_example(): assert session.exit_code == ExitCode.FAILED - products = tree_map(lambda x: x.load(), getattr(session.tasks[0], decorator_name)) - expected = { - 0: tmp_path / "out.txt", - 1: {0: tmp_path / "tuple_out.txt"}, - 2: {0: tmp_path / "list_out.txt"}, - 3: {"a": tmp_path / "dict_out.txt", "b": {"c": tmp_path / "dict_out_2.txt"}}, - } - expected = {decorator_name: expected} + products = tree_map(lambda x: x.load(), getattr(session.tasks[0], arg_name)) + expected = [ + tmp_path / "out.txt", + (tmp_path / "tuple_out.txt",), + [tmp_path / "list_out.txt"], + {"a": tmp_path / "dict_out.txt", "b": {"c": tmp_path / "dict_out_2.txt"}}, + ] + expected = {arg_name: expected} assert products == expected @@ -51,11 +48,12 @@ def task_example(): def test_profile_with_pytree(tmp_path, runner): source = """ import time - import pytask from pytask.tree_util import tree_leaves + from pathlib import Path - @pytask.mark.produces([{"out_1": "out_1.txt"}, {"out_2": "out_2.txt"}]) - def task_example(produces): + def task_example( + produces=[{"out_1": Path("out_1.txt")}, {"out_2": Path("out_2.txt")}] + ): time.sleep(2) for p in tree_leaves(produces): p.write_text("There are nine billion bicycles in Beijing.")