Use Typst's JSON dep file (with outputs)

stepup/reprep/api.py

@@ -378,7 +378,7 @@ def compile_typst(
 
     !!! warning
 
-        This feature will only work well with Typst 0.14 or later.
+        This feature only works with typst 0.15 or newer.
 
         Support for Typst in StepUp RepRep is experimental.
         Expect breaking changes in future releases.
@@ -388,12 +388,6 @@ def compile_typst(
           These images are not rendered, neither are they included in the dep file.
           This currently being addressed in the following issue:
           https://github.com/typst/typst/issues/6858
-        - When the typst compiler detects an error in the input, it doesn't write the dep file.
-          While this is the desirable behavior for Make-like tools, it does not work well in StepUp.
-          This issue is fixed in the main branch of typst, but not yet in a released version:
-          https://github.com/typst/typst/pull/7209
-          After the next release of Typst, StepUp RepRep will be updated to use of JSON dep files:
-          https://github.com/reproducible-reporting/stepup-reprep/pull/22
 
     Parameters
     ----------
@@ -449,7 +443,7 @@ def compile_typst(
         dest = subs(dest)
     if not path_typ.endswith(".typ"):
         raise ValueError(f"The input of the typst command must end with .typ, got {path_typ}.")
-    path_out = make_path_out(path_typ, dest, ".pdf", [".svg", ".png"])
+    path_out = make_path_out(path_typ, dest, ".pdf", [".svg", ".png", ".html"])
 
     stem = path_typ[:-4]
     args = ["compile-typst"]
@@ -462,7 +456,7 @@ def compile_typst(
         paths_out.append(path_out)
     if keep_deps or string_to_bool(getenv("REPREP_KEEP_TYPST_DEPS", "0")):
         args.append("--keep-deps")
-        paths_out.append(f"{stem}.dep")
+        paths_out.append(f"{stem}.deps.json")
     if inventory is None:
         inventory = string_to_bool(getenv("REPREP_TYPST_INVENTORY", "0"))
     if inventory is True:

stepup/reprep/compile_typst.py

@@ -22,18 +22,19 @@
 This wrapper extracts relevant information from a Typst build
 to inform StepUp of input files used or needed.
 
-This is tested with Typst 0.14.
+This is tested with Typst 0.15.
 """
 
 import argparse
 import contextlib
+import json
 import shlex
 import sys
 
 from path import Path, TempDir
 
 from stepup.core.api import amend, getenv
-from stepup.core.utils import filter_dependencies
+from stepup.core.utils import filter_dependencies, string_to_bool
 from stepup.core.worker import WorkThread
 
 from .make_inventory import write_inventory
@@ -47,24 +48,23 @@ def main(argv: list[str] | None = None, work_thread: WorkThread | None = None):
 
     if not args.path_typ.endswith(".typ"):
         raise ValueError("The Typst source must have extension .typ")
-    if not (args.path_out is None or args.path_out.suffix in (".pdf", ".png", ".svg")):
-        raise ValueError("The Typst output must be a PDF, PNG, or SVG file.")
+    if not (args.path_out is None or args.path_out.suffix in (".pdf", ".png", ".svg", ".html")):
+        raise ValueError("The Typst output must be a PDF, PNG, SVG, or HTML file.")
 
-    # Get Typst executable and prepare some arguments that
+    # Prepare the command to run Typst
     if args.typst is None:
         args.typst = getenv("REPREP_TYPST", "typst")
-
-    # Prepare the command to run Typst
     typst_args = [args.typst, "compile", args.path_typ]
     if args.path_out is not None:
         typst_args.append(args.path_out)
     else:
         args.path_out = Path(args.path_typ[:-4] + ".pdf")
     if args.path_out.suffix == ".png":
-        resolution = args.resolution
-        if resolution is None:
-            resolution = int(getenv("REPREP_TYPST_RESOLUTION", "144"))
-        typst_args.append(f"--ppi={resolution}")
+        if args.resolution is None:
+            args.resolution = int(getenv("REPREP_TYPST_RESOLUTION", "144"))
+        typst_args.append(f"--ppi={args.resolution}")
+    elif args.path_out.suffix == ".html":
+        typst_args.append("--features=html")
     for keyval in args.sysinp:
         typst_args.append("--input")
         typst_args.append(keyval)
@@ -75,43 +75,36 @@ def main(argv: list[str] | None = None, work_thread: WorkThread | None = None):
     with contextlib.ExitStack() as stack:
         if args.keep_deps:
             # Remove any existing make-deps output from a previous run.
-            path_dep = Path(args.path_typ[:-4] + ".dep")
-            path_dep.remove_p()
+            path_deps = args.path_typ.with_suffix(".deps.json")
+            path_deps.remove_p()
         else:
             # Use a temporary file for the make-deps output.
-            path_dep = stack.enter_context(TempDir()) / "typst.dep"
-        typst_args.extend(["--deps", path_dep, "--deps-format", "make"])
+            path_deps = stack.enter_context(TempDir()) / "typst.deps.json"
+        typst_args.extend(["--deps", path_deps, "--deps-format", "json"])
 
         # Run typst compile
         returncode, stdout, stderr = work_thread.runsh(shlex.join(typst_args))
         print(stdout)
-        # Get existing input files from the dependency file and amend.
-        # Note that the deps file does not escape colons in paths,
-        # so the code below assumes one never uses colons in paths.
-        inp_paths = []
-        if path_dep.is_file():
-            out_paths = []
-            with open(path_dep) as fh:
-                dep_out, dep_inp = fh.read().split(":", 1)
-                out_paths.extend(shlex.split(dep_out))
-                inp_paths.extend(shlex.split(dep_inp))
+        # Assume there is a single output file, which is the one specified.
+        # This is not correct when there are multiple outputs, e.g. as with SVG and PNG outputs.
+        # Get required input files from the dependency file.
+        if path_deps.is_file():
+            with open(path_deps) as fh:
+                depinfo = json.load(fh)
+            inp_paths = depinfo["inputs"]
+            out_paths = depinfo.get("outputs") or []
         else:
-            print(f"Dependency file not created: {path_dep}.", file=sys.stderr)
-            out_paths = [args.path_out]
+            print(f"Dependency file not created: {path_deps}.", file=sys.stderr)
+            out_paths = []
+            inp_paths = []
 
-    # Look for missing input files in the standard error stream and amend them.
-    if returncode != 0:
-        lead = "error: file not found (searched at "
-        inp_paths.extend(
-            line[len(lead) : -1] for line in stderr.splitlines() if line.startswith(lead)
-        )
     sys.stderr.write(stderr)
     inp_paths = filter_dependencies(inp_paths)
     amend(inp=inp_paths)
 
     # Write inventory
     if args.inventory is not None:
-        inventory_paths = sorted(inp_paths) + out_paths
+        inventory_paths = sorted(inp_paths) + sorted(out_paths)
         write_inventory(args.inventory, inventory_paths, do_amend=False)
 
     # If the output path contains placeholders `{p}`, `{0p}`, or `{t}`,
@@ -149,9 +142,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     parser.add_argument(
         "--keep-deps",
         help="Keep the dependency file after the compilation. "
-        "The default is to use a temporary file, which is removed after it is processed.",
-        action="store_true",
-        default=False,
+        "The default is to use a temporary file, which is removed after it is processed. "
+        "Defaults to the boolean value of ${REPREP_TYPST_KEEP_DEPS}, "
+        "or False if the variable is not defined.",
+        action=argparse.BooleanOptionalAction,
+        default=string_to_bool(getenv("REPREP_TYPST_KEEP_DEPS", "0")),
     )
     parser.add_argument(
         "--inventory",

tests/examples/compile_typst_args/expected_graph.txt

@@ -38,6 +38,7 @@ file:lorem.typ
 step:compile-typst lorem.typ -- --pages 2 --no-pdf-tags
                state = SUCCEEDED
              env_var = REPREP_TYPST [amended]
+                     = REPREP_TYPST_KEEP_DEPS [amended]
                      = STEPUP_PATH_FILTER [amended]
           created by   step:runpy ./plan.py
             consumes   file:./

tests/examples/compile_typst_dep/.gitignore → tests/examples/compile_typst_deps/.gitignore

@@ -1,7 +1,7 @@
 .stepup
 current*
 document.pdf
-document.dep
+document.deps.json
 image.jpg
 document1.pdf
 reproducibility_inventory.txt

tests/examples/compile_typst_dep/expected_graph.txt → tests/examples/compile_typst_deps/expected_graph.txt

@@ -6,7 +6,7 @@ root:
 file:./
                state = STATIC
           created by   root:
-            supplies   file:document.dep
+            supplies   file:document.deps.json
             supplies   file:document.pdf
             supplies   file:document.typ
             supplies   file:image.jpg
@@ -50,17 +50,18 @@ step:compile-typst --keep-deps document.typ
                state = SUCCEEDED
              env_var = REPREP_TYPST [amended]
                      = REPREP_TYPST_ARGS [amended]
+                     = REPREP_TYPST_KEEP_DEPS [amended]
                      = STEPUP_PATH_FILTER [amended]
           created by   step:runpy ./plan.py
             consumes   file:./
             consumes   file:document.typ
             consumes   file:image.jpg [amended]
-             creates   file:document.dep
+             creates   file:document.deps.json
              creates   file:document.pdf
-            supplies   file:document.dep
+            supplies   file:document.deps.json
             supplies   file:document.pdf
 
-file:document.dep
+file:document.deps.json
                state = BUILT
           created by   step:compile-typst --keep-deps document.typ
             consumes   file:./

tests/examples/compile_typst_dep/main.sh → tests/examples/compile_typst_deps/main.sh

@@ -26,7 +26,7 @@ wait
 # Check files that are expected to be present and/or missing.
 [[ -f plan.py ]] || exit 1
 [[ -f document.pdf ]] || exit 1
-[[ -f document.dep ]] || exit 1
+[[ -f document.deps.json ]] || exit 1
 [[ -f image.jpg ]] || exit 1
 [[ -f document1.pdf ]] || exit 1
 [[ -f reproducibility_inventory.txt ]] || exit 1

tests/examples/compile_typst_dep_error/.gitignore → tests/examples/compile_typst_deps_error/.gitignore

@@ -1,4 +1,5 @@
 .stepup
 current*
 document.pdf
+document.deps.json
 data.yaml

tests/examples/compile_typst_dep_error/expected_graph.txt → tests/examples/compile_typst_deps_error/expected_graph.txt

@@ -7,7 +7,7 @@ file:./
                state = STATIC
           created by   root:
             supplies   file:data.yaml
-            supplies   file:document.dep
+            supplies   file:document.deps.json
             supplies   file:document.pdf
             supplies   file:document.typ
             supplies   file:plan.py
@@ -40,26 +40,28 @@ file:document.typ
             supplies   step:compile-typst --keep-deps document.typ
 
 step:compile-typst --keep-deps document.typ
-               state = FAILED
+               state = SUCCEEDED
              env_var = REPREP_TYPST [amended]
                      = REPREP_TYPST_ARGS [amended]
+                     = REPREP_TYPST_KEEP_DEPS [amended]
                      = STEPUP_PATH_FILTER [amended]
           created by   step:runpy ./plan.py
             consumes   file:./
+            consumes   file:data.yaml [amended]
             consumes   file:document.typ
-             creates   file:document.dep
+             creates   file:document.deps.json
              creates   file:document.pdf
-            supplies   file:document.dep
+            supplies   file:document.deps.json
             supplies   file:document.pdf
 
-file:document.dep
-               state = AWAITED
+file:document.deps.json
+               state = BUILT
           created by   step:compile-typst --keep-deps document.typ
             consumes   file:./
             consumes   step:compile-typst --keep-deps document.typ
 
 file:document.pdf
-               state = AWAITED
+               state = BUILT
           created by   step:compile-typst --keep-deps document.typ
             consumes   file:./
             consumes   step:compile-typst --keep-deps document.typ
@@ -76,3 +78,4 @@ file:data.yaml
           created by   step:runsh echo 'fixed: new' > data.yaml
             consumes   file:./
             consumes   step:runsh echo 'fixed: new' > data.yaml
+            supplies   step:compile-typst --keep-deps document.typ

tests/examples/compile_typst_deps_error/expected_stdout.txt

@@ -0,0 +1,20 @@
+        0/0 |    STARTUP │ (Re)initialized boot script
+        0/0 |   DIRECTOR │ Launched worker 0
+        0/1 |      PHASE │ run
+        0/1 |      START │ runpy ./plan.py
+        1/3 |    SUCCESS │ runpy ./plan.py
+        1/3 |      START │ compile-typst --keep-deps document.typ
+        1/3 | RESCHEDULE │ compile-typst --keep-deps document.typ
+──────────────── Rescheduling due to unavailable amended inputs ────────────────
+Missing inputs and/or required directories:
+data.yaml
+────────────────────────────────────────────────────────────────────────────────
+        1/3 |      START │ runsh echo 'fixed: new' > data.yaml
+        2/3 |    SUCCESS │ runsh echo 'fixed: new' > data.yaml
+        2/3 |      START │ compile-typst --keep-deps document.typ
+        3/3 |    SUCCESS │ compile-typst --keep-deps document.typ
+        3/3 |   DIRECTOR │ Trying to delete 0 outdated output(s)
+        3/3 |    WARNING │ Check logs: .stepup/warning.log
+        3/3 |      PHASE │ watch
+        3/3 |   DIRECTOR │ Stopping workers
+        3/3 |   DIRECTOR │ See you!

tests/examples/compile_typst_dep_error/main.sh → tests/examples/compile_typst_deps_error/main.sh

@@ -22,4 +22,6 @@ set +e; wait -fn $PID; RETURNCODE=$?; set -e
 # Check files that are expected to be present and/or missing.
 [[ -f plan.py ]] || exit 1
 [[ -f document.pdf ]] || exit 1
+[[ -f document.deps.json ]] || exit 1
+grep data.yaml document.deps.json
 [[ -f data.yaml ]] || exit 1

tests/examples/compile_typst_error/.gitignore

@@ -1,3 +1,4 @@
 .stepup
 current*
 error.pdf
+error.deps.json