Implement a dependency scanner for LaTeX documents and releases v0.0.9. (#12)

Author: tobiasraabe

.conda/meta.yaml

@@ -20,7 +20,8 @@ requirements:
 
   run:
     - python >=3.6
-    - pytask >=0.0.9
+    - pytask >=0.0.11
+    - latex-dependency-scanner
 
 test:
   requires:

.github/workflows/continuous-integration-workflow.yml

@@ -66,23 +66,3 @@ jobs:
         if: runner.os == 'Linux' && matrix.python-version == '3.8'
         shell: bash -l {0}
         run: cat codecov.yml | curl --data-binary @- https://codecov.io/validate
-
-
-  pre-commit:
-
-    name: Run pre-commit.
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.8
-
-      - name: Install dependencies
-        run: pip install tox
-
-      - name: Run pre-commit
-        run: tox -e pre-commit

CHANGES.rst

@@ -7,9 +7,11 @@ all releases are available on `Anaconda.org
 <https://anaconda.org/pytask/pytask-latex>`_.
 
 
-0.0.9 - 2020-xx-xx
+0.0.9 - 2020-12-28
 ------------------
 
+- :gh:`12` integrates the latex-dependency-scanner to automatically detect dependencies
+  of a LaTeX document and releases v0.0.9.
 - :gh:`13` fixes the CI.
 
 

README.rst

@@ -20,6 +20,10 @@ pytask-latex
 
 pytask-latex allows you to compile LaTeX documents.
 
+It also tries to infer the dependency of the LaTeX document such as included images,
+bibliography files and other .tex files automatically to compile LaTeX documents when it
+is possible.
+
 
 Installation
 ------------
@@ -32,8 +36,9 @@ Install the plugin with
     $ conda install pytask-latex
 
 You also need to have ``latexmk`` installed which determines the necessary number of
-compilation steps. To test whether it is installed, type the following on the command
-line
+compilation steps (`here <https://tex.stackexchange.com/a/249243/194826>`_ is an
+explanation for what latexmk achieves). To test whether it is installed, type the
+following on the command line
 
 .. code-block:: console
 
@@ -190,15 +195,33 @@ to include the latex decorator in the parametrization just like with
 Configuration
 -------------
 
-If you want to change the names of the keys which identify the source file and the
-compiled document, change the following default configuration in your pytask
-configuration file.
+latex_source_key
+    If you want to change the name of the key which identifies the source file, change
+    the following default configuration in your pytask configuration file.
+
+    .. code-block:: ini
+
+        latex_source_key = source
+
+latex_document_key
+    If you want to change the name of the key which identifies the compiled document,
+    change the following default configuration in your pytask configuration file.
+
+    .. code-block:: ini
+
+        latex_source_key = source
 
-.. code-block:: ini
+infer_latex_dependencies
+    pytask-latex tries to scan your LaTeX document for included files with the help of
+    `latex-dependency-scanner <https://github.com/pytask-dev/latex-dependency-scanner>`_
+    if the following configuration value is true which is also the default.
 
-    latex_source_key = source
-    latex_document_key = document
+        infer_latex_dependencies = true
 
+    Since the package is in its early development phase and LaTeX provides a myriad of
+    ways to include files as well as providing shortcuts for paths (e.g.,
+    ``\graphicspath``), there are definitely some rough edges left. File an issue here
+    or in the other project to make us aware of the problem.
 
 
 Changes

environment.yml

@@ -3,7 +3,7 @@ channels:
   - conda-forge
   - pytask
 dependencies:
-  - python=3.7
+  - python = 3.8
   - pip
 
   # Conda
@@ -12,8 +12,9 @@ dependencies:
   - conda-verify
 
   # Package dependencies
-  - pytask >= 0.0.9
+  - pytask >= 0.0.11
   - pytask-parallel >= 0.0.4
+  - latex-dependency-scanner
 
   # Misc
   - black

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.8
+current_version = 0.0.9
 parse = (?P<major>\d+)\.(?P<minor>\d+)(\.(?P<patch>\d+))(\-?((dev)?(?P<dev>\d+))?)
 serialize = 
 	{major}.{minor}.{patch}dev{dev}

setup.py

@@ -3,7 +3,7 @@
 
 setup(
     name="pytask-latex",
-    version="0.0.8",
+    version="0.0.9",
     packages=find_packages(where="src"),
     package_dir={"": "src"},
     entry_points={"pytask": ["pytask_latex = pytask_latex.plugin"]},

src/pytask_latex/__init__.py

@@ -1 +1 @@
-__version__ = "0.0.8"
+__version__ = "0.0.9"

src/pytask_latex/collect.py

@@ -11,9 +11,12 @@
 from _pytask.config import hookimpl
 from _pytask.mark import get_specific_markers_from_task
 from _pytask.mark import has_marker
+from _pytask.mark import Mark
+from _pytask.nodes import _collect_nodes
 from _pytask.nodes import FilePathNode
 from _pytask.nodes import PythonFunctionTask
 from _pytask.parametrize import _copy_func
+from latex_dependency_scanner import scan
 
 
 DEFAULT_OPTIONS = ["--pdf", "--interaction=nonstopmode", "--synctex=1", "--cd"]
@@ -93,6 +96,9 @@ def pytask_collect_task_teardown(session, task):
 
         task.function = latex_function
 
+        if session.config["infer_latex_dependencies"]:
+            task = _add_latex_dependencies_retroactively(task, session)
+
 
 def _get_node_from_dictionary(obj, key, fallback=0):
     if isinstance(obj, Path):
@@ -102,6 +108,54 @@ def _get_node_from_dictionary(obj, key, fallback=0):
     return obj
 
 
+def _add_latex_dependencies_retroactively(task, session):
+    """Add dependencies from LaTeX document to task.
+
+    Unfortunately, the dependencies have to be added retroactively, after the task has
+    been created, since one needs access to the LaTeX document which is validated after
+    the task is created.
+
+    1. Collect all possible files from the LaTeX document.
+    2. Keep only files which exist or which are produced by some other task. Note that
+       only tasks are considered which have been collected until this point.
+    3. Mark the task such that it will be evaluated at last.
+
+    Parameters
+    ----------
+    task
+        The LaTeX task.
+    session : _pytask.session.Session
+        The session.
+
+    """
+    source = _get_node_from_dictionary(
+        task.depends_on, session.config["latex_source_key"]
+    )
+
+    # Scan the LaTeX document for included files.
+    latex_dependencies = set(scan(source.path))
+
+    # Remove duplicated dependencies which have already been added by the user.
+    existing_paths = {
+        i.path for i in task.depends_on.values() if isinstance(i, FilePathNode)
+    }
+    new_dependencies = latex_dependencies - existing_paths
+
+    used_integer_keys = [i for i in task.depends_on if isinstance(i, int)]
+    max_int = max(used_integer_keys) if used_integer_keys else 0
+
+    new_dependencies = dict(enumerate(new_dependencies, max_int + 1))
+    collected_dependencies = _collect_nodes(
+        session, task.path, task.name, new_dependencies
+    )
+    task.depends_on = {**task.depends_on, **collected_dependencies}
+
+    # Mark the task as being delayed to avoid conflicts with unmatched dependencies.
+    task.markers.append(Mark("try_last", (), {}))
+
+    return task
+
+
 def _merge_all_markers(task):
     """Combine all information from markers for the compile latex function."""
     latex_marks = get_specific_markers_from_task(task, "latex")

src/pytask_latex/config.py

@@ -1,5 +1,7 @@
 """Configure pytask."""
 from _pytask.config import hookimpl
+from _pytask.shared import convert_truthy_or_falsy_to_bool
+from _pytask.shared import get_first_non_none_value
 
 
 @hookimpl
@@ -10,3 +12,9 @@ def pytask_parse_config(config, config_from_file):
     config["latex_document_key"] = config_from_file.get(
         "latex_document_key", "document"
     )
+    config["infer_latex_dependencies"] = get_first_non_none_value(
+        config_from_file,
+        key="infer_latex_dependencies",
+        callback=convert_truthy_or_falsy_to_bool,
+        default=True,
+    )

tests/test_collect.py

@@ -91,20 +91,35 @@ def test_pytask_collect_task(name, expected):
 )
 @pytest.mark.parametrize("latex_source_key", ["source", "script", "main"])
 @pytest.mark.parametrize("latex_document_key", ["document", "compiled_doc"])
+@pytest.mark.parametrize("infer_latex_dependencies", [True, False])
 def test_pytask_collect_task_teardown(
-    depends_on, produces, expectation, latex_source_key, latex_document_key
+    tmp_path,
+    depends_on,
+    produces,
+    expectation,
+    latex_source_key,
+    latex_document_key,
+    infer_latex_dependencies,
 ):
+    if infer_latex_dependencies:
+        tmp_path.joinpath(depends_on[0]).touch()
+
     session = DummyClass()
     session.config = {
         "latex_source_key": latex_source_key,
         "latex_document_key": latex_document_key,
+        "infer_latex_dependencies": infer_latex_dependencies,
     }
 
     task = DummyClass()
+    task.path = tmp_path / "task_dummy.py"
+    task.name = tmp_path.as_posix() + "task_dummy.py::task_dummy"
     task.depends_on = {
-        i: FilePathNode.from_path(Path(n)) for i, n in enumerate(depends_on)
+        i: FilePathNode.from_path(tmp_path / n) for i, n in enumerate(depends_on)
+    }
+    task.produces = {
+        i: FilePathNode.from_path(tmp_path / n) for i, n in enumerate(produces)
     }
-    task.produces = {i: FilePathNode.from_path(Path(n)) for i, n in enumerate(produces)}
     task.markers = [Mark("latex", (), {})]
     task.function = task_dummy
     task.function.pytaskmark = task.markers
@@ -116,12 +131,7 @@ def test_pytask_collect_task_teardown(
 @pytest.mark.unit
 @pytest.mark.parametrize(
     "obj, key, expected",
-    [
-        (1, "asds", 1),
-        (1, None, 1),
-        ({"a": 1}, "a", 1),
-        ({0: 1}, "a", 1),
-    ],
+    [(1, "asds", 1), (1, None, 1), ({"a": 1}, "a", 1), ({0: 1}, "a", 1)],
 )
 def test_get_node_from_dictionary(obj, key, expected):
     result = _get_node_from_dictionary(obj, key)

tests/test_latex_dependency_scanner.py

@@ -0,0 +1,38 @@
+import textwrap
+
+import pytest
+from conftest import needs_latexmk
+from conftest import skip_on_github_actions_with_win
+from pytask import main
+
+
+@needs_latexmk
+@skip_on_github_actions_with_win
+@pytest.mark.end_to_end
+def test_infer_dependencies_from_task(tmp_path):
+    task_source = """
+    import pytask
+
+    @pytask.mark.latex
+    @pytask.mark.depends_on("document.tex")
+    @pytask.mark.produces("document.pdf")
+    def task_compile_document():
+        pass
+
+    """
+    tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source))
+
+    latex_source = r"""
+    \documentclass{report}
+    \begin{document}
+    \input{sub_document}
+    \end{document}
+    """
+    tmp_path.joinpath("document.tex").write_text(textwrap.dedent(latex_source))
+    tmp_path.joinpath("sub_document.tex").write_text("Lorem ipsum.")
+
+    session = main({"paths": tmp_path})
+
+    assert session.exit_code == 0
+    assert len(session.tasks) == 1
+    assert len(session.tasks[0].depends_on) == 2

tox.ini

@@ -9,8 +9,9 @@ basepython = python
 
 [testenv:pytest]
 conda_deps =
-    pytask >=0.0.9
+    pytask >=0.0.11
     pytask-parallel >=0.0.4
+    latex-dependency-scanner
     pytest
     pytest-cov
     pytest-xdist