Add pytask_add_hooks to register plugins and hooks. (#5)

Author: tobiasraabe

.conda/meta.yaml

@@ -18,10 +18,12 @@ build:
 requirements:
   host:
     - python
+    - pip
     - setuptools
 
   run:
     - python >=3.6
+    - setuptools
     - attrs >=17.4.0
     - click
     - networkx
@@ -30,6 +32,8 @@ requirements:
     - pytest
 
 test:
+  imports:
+    - pytask
   requires:
     - pexpect
     - pytest
@@ -45,6 +49,6 @@ test:
 about:
   home: https://github.com/pytask-dev/pytask
   license: none
-  summary: In its highest aspirations, **pytask** tries to be pytest as a build system.
+  summary: In its highest aspirations, pytask tries to be pytest as a build system.
   doc_url: https://pytask-dev.readthedocs.io/en/latest
   dev_url: https://github.com/pytask-dev/pytask/

.github/ISSUE_TEMPLATE/documentation.md

@@ -12,7 +12,7 @@ labels: "documentation"
 Provide the location of the documentation, e.g. an URL of the documentation.
 
 **Note**: You can check the latest versions of the docs on `main`
-[here](https://pytask.readthedocs.io/en/latest).
+[here](https://pytask-dev.readthedocs.io/en/latest).
 
 #### Documentation problem
 

.github/workflows/continuous-integration-workflow.yml

@@ -35,7 +35,7 @@ jobs:
 
       - name: Run unit tests and doctests.
         shell: bash -l {0}
-        run: tox -e pytest -- -m "unit or (not integration and not end_to_end)" --cov=./ --cov-report=xml
+        run: tox -e pytest -- -m "unit or (not integration and not end_to_end)" --cov=./ --cov-report=xml -n auto
 
       - name: Upload coverage report for unit tests and doctests.
         if: runner.os == 'Linux' && matrix.python-version == '3.8'
@@ -44,7 +44,7 @@ jobs:
 
       - name: Run integration tests.
         shell: bash -l {0}
-        run: tox -e pytest -- -m integration --cov=./ --cov-report=xml
+        run: tox -e pytest -- -m integration --cov=./ --cov-report=xml -n auto
 
       - name: Upload coverage reports of integration tests.
         if: runner.os == 'Linux' && matrix.python-version == '3.8'
@@ -53,7 +53,7 @@ jobs:
 
       - name: Run end-to-end tests.
         shell: bash -l {0}
-        run: tox -e pytest -- -m end_to_end --cov=./ --cov-report=xml
+        run: tox -e pytest -- -m end_to_end --cov=./ --cov-report=xml -n auto
 
       - name: Upload coverage reports of end-to-end tests.
         if: runner.os == 'Linux' && matrix.python-version == '3.8'

README.rst

@@ -46,8 +46,8 @@ include:
 Installation
 ------------
 
-**pytask** is available on `Anaconda.org <https://anaconda.org/pytask/pytask>`_. Install
-the package with
+pytask is available on `Anaconda.org <https://anaconda.org/pytask/pytask>`_. Install the
+package with
 
 .. code-block:: bash
 

docs/changes.rst

@@ -12,6 +12,7 @@ all releases are available on `Anaconda.org <https://anaconda.org/pytask/pytask>
 - :gh:`2` provided multiple small changes.
 - :gh:`3` implements a class which holds the execution report of one task.
 - :gh:`4` makes adjustments after moving to ``main`` as the default branch.
+- :gh:`5` adds ``pytask_add_hooks`` to add more hook specifications and register hooks.
 
 
 0.0.1 - 2020-06-29

docs/index.rst

@@ -10,6 +10,10 @@ pytask
 .. image:: https://readthedocs.org/projects/pytask-dev/badge/?version=latest
     :target: https://pytask-dev.readthedocs.io/en/latest
 
+.. image:: https://github.com/pytask-dev/pytask/workflows/Continuous%20Integration%20Workflow/badge.svg?branch=main
+    :target: https://github.com/pytask-dev/pytask/actions?query=branch%3Amain
+
+
 .. image:: https://codecov.io/gh/pytask-dev/pytask/branch/main/graph/badge.svg
     :target: https://codecov.io/gh/pytask-dev/pytask
 

environment.yml

@@ -18,6 +18,7 @@ dependencies:
   - pytest
 
   # Misc
+  - bump2version
   - jupyterlab
   - matplotlib
   - pdbpp

setup.cfg

@@ -2,8 +2,8 @@
 current_version = 0.0.1
 parse = (?P<major>\d+)\.(?P<minor>\d+)(\.(?P<patch>\d+))(\-?((dev)?(?P<dev>\d+))?)
 serialize =
-    {major}.{minor}.{patch}dev{dev}
-    {major}.{minor}.{patch}
+	{major}.{minor}.{patch}dev{dev}
+	{major}.{minor}.{patch}
 
 [bumpversion:file:setup.py]
 

src/pytask/cli.py

@@ -2,8 +2,7 @@
 from pathlib import Path
 
 import click
-from pytask import debugging
-from pytask import hookimpl
+import pytask
 from pytask.main import main
 from pytask.pluginmanager import get_plugin_manager
 
@@ -15,17 +14,26 @@ def add_parameters(func):
     """Add parameters from plugins to the commandline interface."""
     pm = get_plugin_manager()
     pm.register(sys.modules[__name__])
-    pm.register(debugging)
+    pm.hook.pytask_add_hooks(pm=pm)
     pm.hook.pytask_add_parameters_to_cli(command=func)
 
     return func
 
 
+@pytask.hookimpl
+def pytask_add_hooks(pm):
+    from pytask import database
+    from pytask import debugging
+
+    pm.register(database)
+    pm.register(debugging)
+
+
 def _to_path(ctx, param, value):  # noqa: U100
     return [Path(i).resolve() for i in value]
 
 
-@hookimpl
+@pytask.hookimpl
 def pytask_add_parameters_to_cli(command):
     additional_parameters = [
         click.Argument(
@@ -38,24 +46,6 @@ def pytask_add_parameters_to_cli(command):
             help="Ignore path (globs and multi allowed).",
         ),
         click.Option(["--debug-pytask"], is_flag=True, help="Debug pytask."),
-        click.Option(
-            ["--database-provider"],
-            type=click.Choice(["sqlite", "postgres", "mysql", "oracle", "cockroach"]),
-            help="Database provider.",
-        ),
-        click.Option(
-            ["--database-filename"], type=click.Path(), help="Path to database.",
-        ),
-        click.Option(
-            ["--database-create-db"],
-            type=bool,
-            help="Create database if it does not exist.",
-        ),
-        click.Option(
-            ["--database-create-tables"],
-            type=bool,
-            help="Create tables if they do not exist.",
-        ),
     ]
     command.params.extend(additional_parameters)
 

src/pytask/config.py

@@ -8,9 +8,9 @@
 
 import click
 import pytask
+from pytask.shared import get_first_not_none_value
 from pytask.shared import to_list
 
-
 IGNORED_FILES_AND_FOLDERS = [
     "*/.git/*",
     "*/__pycache__/*",
@@ -23,7 +23,7 @@
 def pytask_configure(pm, config_from_cli):
     config = {"pm": pm, "terminal_width": _get_terminal_width()}
 
-    paths = _get_first_not_none_value(config_from_cli, key="paths", callback=to_list)
+    paths = get_first_not_none_value(config_from_cli, key="paths", callback=to_list)
     paths = [Path(p).resolve() for path in paths for p in glob.glob(path.as_posix())]
     config["paths"] = paths if paths else [Path.cwd().resolve()]
 
@@ -44,7 +44,7 @@ def pytask_configure(pm, config_from_cli):
 @pytask.hookimpl
 def pytask_parse_config(config, config_from_cli, config_from_file):
     config["ignore"] = (
-        _get_first_not_none_value(
+        get_first_not_none_value(
             config_from_cli,
             config_from_file,
             key="ignore",
@@ -54,35 +54,13 @@ def pytask_parse_config(config, config_from_cli, config_from_file):
         + IGNORED_FILES_AND_FOLDERS
     )
 
-    config["debug_pytask"] = _get_first_not_none_value(
+    config["debug_pytask"] = get_first_not_none_value(
         config_from_cli, config_from_file, key="debug_pytask", default=False
     )
     if config["debug_pytask"]:
         config["pm"].trace.root.setwriter(click.echo)
         config["pm"].enable_tracing()
 
-    provider = _get_first_not_none_value(
-        config_from_cli, config_from_file, key="database_provider", default="sqlite"
-    )
-    filename = _get_first_not_none_value(
-        config_from_cli,
-        config_from_file,
-        key="database_filename",
-        default=".pytask.sqlite3",
-    )
-    create_db = _get_first_not_none_value(
-        config_from_cli, config_from_file, key="database_create_db", default=True
-    )
-    create_tables = _get_first_not_none_value(
-        config_from_cli, config_from_file, key="database_create_tables", default=True
-    )
-    config["database"] = {
-        "provider": provider,
-        "filename": Path(config["root"], filename).resolve().as_posix(),
-        "create_db": create_db,
-        "create_tables": create_tables,
-    }
-
 
 def _find_project_root_and_ini(paths):
     try:
@@ -126,42 +104,11 @@ def _read_config(path):
     return dict(config["pytask"])
 
 
-def _get_first_not_none_value(*configs, key, default=None, callback=None):
-    """Get the first non-None value for a key from a list of dictionaries.
-
-    This function allows to prioritize information from many configurations by changing
-    the order of the inputs while also providing a default.
-
-    Examples
-    --------
-    >>> _get_first_not_none_value({"a": None}, {"a": 1}, key="a")
-    1
-
-    >>> _get_first_not_none_value({"a": None}, {"a": None}, key="a", default="default")
-    'default'
-
-    >>> _get_first_not_none_value({}, {}, key="a", default="default")
-    'default'
-
-    >>> _get_first_not_none_value({"a": None}, {"a": "b"}, key="a", callback=to_list)
-    ['b']
-
-    """
-    return next(
-        (
-            config[key] if callback is None else callback(config[key])
-            for config in configs
-            if config.get(key, None) is not None
-        ),
-        default,
-    )
-
-
 def _get_terminal_width() -> int:
     """Get the window width of the terminal."""
     width, _ = shutil.get_terminal_size(fallback=(80, 24))
 
-    # The Windows get_terminal_size may be bogus, let's sanify a bit.
+    # The Windows get_terminal_size may be bogus, let's sanitize a bit.
     if width < 40:
         width = 80
 

src/pytask/database.py

@@ -1,4 +1,9 @@
+from pathlib import Path
+
+import click
+import pytask
 from pony import orm
+from pytask.shared import get_first_not_none_value
 
 
 db = orm.Database()
@@ -28,3 +33,57 @@ def create_or_update_state(first_key, second_key, state):
         State(task=first_key, node=second_key, state=state)
     else:
         state_in_db.state = state
+
+
+@pytask.hookimpl
+def pytask_add_parameters_to_cli(command):
+    additional_parameters = [
+        click.Option(
+            ["--database-provider"],
+            type=click.Choice(["sqlite", "postgres", "mysql", "oracle", "cockroach"]),
+            help="Database provider.  [default: sqlite]",
+        ),
+        click.Option(
+            ["--database-filename"], type=click.Path(), help="Path to database.",
+        ),
+        click.Option(
+            ["--database-create-db"],
+            type=bool,
+            help="Create database if it does not exist.",
+        ),
+        click.Option(
+            ["--database-create-tables"],
+            type=bool,
+            help="Create tables if they do not exist.",
+        ),
+    ]
+    command.params.extend(additional_parameters)
+
+
+@pytask.hookimpl
+def pytask_parse_config(config, config_from_cli, config_from_file):
+    provider = get_first_not_none_value(
+        config_from_cli, config_from_file, key="database_provider", default="sqlite"
+    )
+    filename = get_first_not_none_value(
+        config_from_cli,
+        config_from_file,
+        key="database_filename",
+        default=".pytask.sqlite3",
+    )
+    filename = Path(filename)
+    if not filename.is_absolute():
+        filename = Path(config["root"], filename).resolve().as_posix()
+
+    create_db = get_first_not_none_value(
+        config_from_cli, config_from_file, key="database_create_db", default=True
+    )
+    create_tables = get_first_not_none_value(
+        config_from_cli, config_from_file, key="database_create_tables", default=True
+    )
+    config["database"] = {
+        "provider": provider,
+        "filename": filename,
+        "create_db": create_db,
+        "create_tables": create_tables,
+    }

src/pytask/debugging.py

@@ -4,24 +4,33 @@
 
 import click
 import pytask
-from pytask.config import _get_first_not_none_value
 from pytask.nodes import PythonFunctionTask
+from pytask.shared import get_first_not_none_value
 
 
 @pytask.hookimpl
 def pytask_add_parameters_to_cli(command):
     additional_parameters = [
-        click.Option(["--pdb"], help="Enter debugger on errors.", is_flag=True),
-        click.Option(["--trace"], help="Enter debugger at test start.", is_flag=True),
+        click.Option(
+            ["--pdb"], help="Enter debugger on errors.", is_flag=True, default=None
+        ),
+        click.Option(
+            ["--trace"],
+            help="Enter debugger when starting each task.",
+            is_flag=True,
+            default=None,
+        ),
     ]
     command.params.extend(additional_parameters)
 
 
 @pytask.hookimpl
-def pytask_parse_config(config, config_from_cli):
-    config["pdb"] = _get_first_not_none_value(config_from_cli, key="pdb", default=False)
-    config["trace"] = _get_first_not_none_value(
-        config_from_cli, key="trace", default=False
+def pytask_parse_config(config, config_from_cli, config_from_file):
+    config["pdb"] = get_first_not_none_value(
+        config_from_cli, config_from_file, key="pdb", default=False
+    )
+    config["trace"] = get_first_not_none_value(
+        config_from_cli, config_from_file, key="trace", default=False
     )