fix cmake use of workspace.view

Author: tjfulle

.github/workflows/workflow.yml

@@ -28,11 +28,11 @@ jobs:
       - name: Pytest
         run: |
           pip install .[dev]
-          pytest src/canary_gitlab/tests
           pytest src/canary_cmake/tests
-          pytest src/canary_vvtest/tests
-          pytest src/canary_pyt/tests
+          pytest src/canary_gitlab/tests
           pytest src/canary_hpc/tests
+          pytest src/canary_pyt/tests
+          pytest src/canary_vvtest/tests
           pytest tests
 #      - name: Coverage
 #        run: |

pyproject.toml

@@ -106,10 +106,11 @@ max_supported_python = "3.14"
 [tool.pytest.ini_options]
 testpaths = [
     "tests",
-    "src/canary_vvtest/tests",
     "src/canary_cmake/tests",
+    "src/canary_gitlab/tests",
     "src/canary_hpc/tests",
-    "src/canary_gitlab/tests"
+    "src/canary_pyt/tests",
+    "src/canary_vvtest/tests",
 ]
 norecursedirs = "mock data generators"
 addopts = "-ra --durations=10"
@@ -124,7 +125,7 @@ output = "coverage.xml"
 branch = true
 omit = [ "*/third_party/*" ]
 source = [ "_canary" ]
-command_line = "-m pytest ./tests"
+command_line = "-m pytest ./tests ./src/canary_cmake/tests ./src/canary_gitlab/tests ./src/canary_hpc/tests ./src/canary_pyt/tests ./src/canary_vvtest/tests"
 
 [tool.coverage.html]
 directory = "html.cov"

src/_canary/plugins/subcommands/check.py

@@ -9,6 +9,11 @@
 import site
 import subprocess
 import sys
+import time
+from concurrent.futures import Future
+from concurrent.futures import ProcessPoolExecutor
+from concurrent.futures import as_completed
+from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING
 from typing import Any
@@ -31,6 +36,14 @@ def canary_addcommand(parser: "Parser") -> None:
 stderr: Any = subprocess.PIPE
 
 logger = logging.get_logger(__name__)
+test_paths = (
+    "tests",
+    "src/canary_cmake/tests",
+    "src/canary_gitlab/tests",
+    "src/canary_hpc/tests",
+    "src/canary_pyt/tests",
+    "src/canary_vvtest/tests",
+)
 
 
 class Action(argparse.Action):
@@ -54,8 +67,8 @@ def setup_parser(self, parser: "Parser") -> None:
         parser.add_argument(
             "-b", nargs=0, action=Action, help="run bandit security checks (default)"
         )
-        parser.add_argument("-t", nargs=0, action=Action, help="run pytest")
-        parser.add_argument("-C", nargs=0, action=Action, help="run coverage (default)")
+        parser.add_argument("-t", nargs=0, action=Action, help="run pytest (default)")
+        parser.add_argument("-C", nargs=0, action=Action, help="run coverage")
         parser.add_argument("-e", nargs=0, action=Action, help="run examples test")
         parser.add_argument("-d", nargs=0, action=Action, help="make docs")
         parser.add_argument("-v", action="store_true", help="verbose")
@@ -72,7 +85,7 @@ def execute(self, args: argparse.Namespace) -> int:
             raise ValueError("canary check must be run from a editable install of canary")
         self.root = os.path.normpath(str(root))
         if not hasattr(args, "action"):
-            args.action = set("fcmbC")
+            args.action = set("fcmbt")
         if shutil.which("ruff") is None and "f" in args.action:
             raise ValueError("ruff must be on PATH to format and check code")
         if shutil.which("ruff") is None and "c" in args.action:
@@ -179,24 +192,20 @@ def run_tests(self, args: argparse.Namespace):
         if "e" in args.action:
             os.environ["CANARY_RUN_EXAMPLES_TEST"] = "1"
         with working_dir(self.root):
-            if "C" not in args.action:
-                pm = logger.progress_monitor(f"Running tests in {self.root}/tests")
-                pytest("./tests")
-                pm.done()
-                pm = logger.progress_monitor(f"Running tests in {self.root}/canary_pyt/tests")
-                pytest("./src/canary_pyt/tests")
-                pm.done()
-                pm = logger.progress_monitor(f"Running tests in {self.root}/canary_cmake/tests")
-                pytest("./src/canary_cmake/tests")
-                pm.done()
-                pm = logger.progress_monitor(f"Running tests in {self.root}/canary_hpc/tests")
-                pytest("./src/canary_hpc/tests")
-                pm.done()
-                pm = logger.progress_monitor(f"Running tests in {self.root}/canary_vvtest/tests")
-                pytest("./src/canary_vvtest/tests")
-                pm.done()
+            if "t" in args.action:
+                results = run_pytests_parallel(Path(self.root), test_paths)
+                failed = [r for r in results if not r.ok]
+                if failed:
+                    for r in failed:
+                        if r.stdout:
+                            sys.stdout.write(r.stdout)
+                        if r.stderr:
+                            sys.stderr.write(r.stderr)
+                    raise ValueError(
+                        f"{len(failed)} pytest runs failed: {', '.join(r.path for r in failed)}"
+                    )
             else:
-                pm = logger.progress_monitor(f"Running coverage in {self.root}/tests")
+                pm = logger.progress_monitor(f"Running coverage in {self.root}")
                 coverage("run")
                 pm.done()
                 pm = logger.progress_monitor("Creating coverage report")
@@ -302,6 +311,56 @@ def pytest(*args: str, **kwargs: Any) -> subprocess.CompletedProcess:
     return cp
 
 
+@dataclass(frozen=True)
+class PytestResult:
+    path: str
+    returncode: int
+    stdout: str
+    stderr: str
+    elapsed_s: float
+
+    @property
+    def ok(self) -> bool:
+        return self.returncode == 0
+
+
+def run_pytest_one(root: str, relpath: str, pytest_args: tuple[str, ...] = ()) -> PytestResult:
+    # Runs in a worker process
+    t0 = time.time()
+    command = ["pytest", relpath, *pytest_args]
+    cp = subprocess.run(command, cwd=root, stdout=stdout, stderr=stderr, encoding="utf-8")
+    return PytestResult(
+        path=relpath,
+        returncode=cp.returncode,
+        stdout=cp.stdout or "",
+        stderr=cp.stderr or "",
+        elapsed_s=time.time() - t0,
+    )
+
+
+def run_pytests_parallel(
+    root: Path,
+    test_paths: tuple[str, ...],
+    *,
+    max_workers: int | None = None,
+    pytest_args: tuple[str, ...] = (),
+) -> list[PytestResult]:
+    results: list[PytestResult] = []
+
+    with ProcessPoolExecutor(max_workers=max_workers or os.cpu_count()) as ex:
+        futures: dict[Future, str] = {}
+        for p in test_paths:
+            logger.info(f"Submitting tests in {p} to pytest")
+            fut = ex.submit(run_pytest_one, str(root), str(p), pytest_args)
+            futures[fut] = str(p)
+        for fut in as_completed(futures):
+            res = fut.result()
+            results.append(res)
+            # Print on completion (no interleaving during run)
+            logger.info(f"pytest finished: {res.path} ({res.elapsed_s:.1f}s) rc={res.returncode}")
+    return results
+
+
 def coverage(*args: str, **kwargs: Any) -> subprocess.CompletedProcess:
     kwargs["stdout"] = stdout
     kwargs["stderr"] = stderr

src/canary_cmake/cdash/__init__.py

@@ -52,12 +52,6 @@ def setup_parser(self, parser: "canary.Parser"):
             metavar="directory",
             help="Write reports to this directory [default: $session/_reports/cdash]",
         )
-        p.add_argument(
-            "-j",
-            dest="json",
-            metavar="file",
-            help="Create reports from this JSON file [default: $session/_reports/cdash]",
-        )
         group = p.add_mutually_exclusive_group()
         group.add_argument(
             "--track",
@@ -232,11 +226,7 @@ def make_gitlab_issues(self, session: "canary.Session | None" = None, **kwargs:
         return
 
     def create(self, **kwargs: Any) -> None:
-        reporter: CDashXMLReporter
-        if kwargs.get("json"):
-            reporter = CDashXMLReporter.from_json(file=kwargs["json"], dest=kwargs["dest"])
-        else:
-            reporter = CDashXMLReporter.from_workspace(dest=kwargs["dest"])
+        reporter: CDashXMLReporter = CDashXMLReporter.from_workspace(dest=kwargs["dest"])
         if kwargs["f"]:
             opts = dict(
                 buildstamp=kwargs["buildstamp"],

src/canary_cmake/cdash/xmlreporter.py

@@ -34,48 +34,18 @@ def __init__(self, dest: str | None = None) -> None:
 
     @classmethod
     def from_workspace(cls, dest: str | None = None) -> "CDashXMLReporter":
-        workspace = canary.Workspace.load()
+        workspace: canary.Workspace = canary.Workspace.load()
         jobs = workspace.load_jobs()
         if not jobs:
             raise ValueError(f"No results found in {workspace.root}")
         if dest is None:
-            dest = str((workspace.view or workspace.sessions_dir) / "CDASH")
+            view = workspace.latest_view()
+            dest = str((workspace.sessions_dir if view is None else view.dir) / "CDASH")
         self = cls(dest=dest)
         for job in jobs:
             self.data.add_job(job)
         return self
 
-    @classmethod
-    def from_json(cls, file: str, dest: str | None = None) -> "CDashXMLReporter":
-        """Create an xml report from a json report"""
-        raise NotImplementedError("No way of loading the job directly from lock yet")
-
-    #        from _canary.testcase import factory as testcase_factory
-    #
-    #        dest = dest or os.path.join(os.path.dirname(file), "CDASH")
-    #        self = cls(dest=dest)
-    #        data = json.load(open(file))
-    #        ts: TopologicalSorter = TopologicalSorter()
-    #        for id, state in data.items():
-    #            for name, value in state["properties"].items():
-    #                if name == "dependencies":
-    #                    dependencies = value
-    #                    dep_ids = [d["properties"]["id"] for d in dependencies]
-    #                    ts.add(id, *dep_ids)
-    #                    break
-    #        jobs: dict[str, canary.Job] = {}
-    #        for id in ts.static_order():
-    #            state = data[id]
-    #            job = testcase_factory(state.pop("type"))
-    #            job.setstate(state)
-    #            for i, dep in enumerate(job.dependencies):
-    #                job.dependencies[i] = jobs[dep.id]
-    #            jobs[id] = job
-    #        for job in jobs.values():
-    #            # job.refresh()
-    #            self.data.add_job(job)
-    #        return self
-
     def create(
         self,
         buildname: str,