feat: add subdir for pytest param id with config artifacts_use_subdir_for_parametrize (#12)

* feat: add subdir for pytest param id with config `artifacts_use_subdir_for_parametrize`
* docs: add config table and exmaples

Author: ketozhang

README.md

@@ -46,24 +46,60 @@ def test_benchmark(artifacts):
 The test case directory is named after the test path, function name, and if any, the test parameter ID.
 
 ### Configure
-Configurations may be set in `pyproject.toml`, `pytest.ini`, or passed as command line options.
+Configurations may be set in `pyproject.toml` or `pytest.ini`. Some options can also be set via CLI (use `pytest --help`)
+
+| Option | Type | Default | Description |
+| --- | --- | --- | --- |
+| `artifacts_dir` | str | `.artifacts/` | Directory to store test artifacts. Also settable via the `--artifacts-dir` CLI option, which takes precedence over the ini setting. |
+| `artifacts_use_subdir_for_parametrize` | bool | `false` | When `True`, parametrized tests get a subdirectory per parameter ID (e.g. `.artifacts/test_foo/param_id/`). When `False`, all parameter variants share the same `.artifacts/test_foo/` directory and overwrite each other. |
 
 ```toml
 # pyproject.toml
-[tool.pytest]
-artifacts_dir = .artifacts/
+[tool.pytest.ini_options]
+artifacts_dir = ".artifacts/"
+artifacts_use_subdir_for_parametrize = false
 ```
 
 ```ini
 # pytest.ini
 [pytest]
 artifacts_dir = .artifacts/
+artifacts_use_subdir_for_parametrize = false
 ```
 
 ```sh
 pytest --artifacts-dir .artifacts/ tests/
 ```
 
+#### Parametrized test layout
+
+
+```py
+@pytest.mark.parametrize("x", [1, 2])
+def test_foo(artifacts, x):
+    with artifacts.open("out.txt", "w") as f:
+        f.write(str(x))
+```
+
+With `artifacts_use_subdir_for_parametrize = false`:
+```
+.artifacts/
+└── test_foo[1]/
+    └── out.txt
+└── test_foo[2]/
+    └── out.txt
+```
+
+With `artifacts_use_subdir_for_parametrize = true`:
+```
+.artifacts/
+└── test_foo/
+    ├── 1/
+    │   └── out.txt
+    └── 2/
+        └── out.txt
+```
+
 ## Contributing
 
 Contributions are very welcome.

src/pytest_artifacts/plugin.py

@@ -23,6 +23,12 @@ def pytest_addoption(parser):
     parser.addini(
         "artifacts_dir", "Directory to store test artifacts.", default=".artifacts/"
     )
+    parser.addini(
+        "artifacts_use_subdir_for_parametrize",
+        "Whether to use subdirectories for parameterized tests.",
+        default=False,
+        type="bool",
+    )
 
 
 def pytest_configure(config):
@@ -32,9 +38,16 @@ def pytest_configure(config):
 
     config.artifacts_dir = artifacts_dir
 
+    artifacts_use_subdir_for_parametrize = config.getini(
+        "artifacts_use_subdir_for_parametrize"
+    )
+    config.artifacts_use_subdir_for_parametrize = artifacts_use_subdir_for_parametrize
+
 
 @pytest.fixture
-def artifacts(request) -> Generator[ArtifactsRepository, None, None]:  # pylint: disable=invalid-name
+def artifacts(
+    request: pytest.FixtureRequest,
+) -> Generator[ArtifactsRepository, None, None]:  # pylint: disable=invalid-name
     """Provide an artifact repository to store and access test artifacts for the
     particular test case.
 
@@ -50,10 +63,21 @@ def artifacts(request) -> Generator[ArtifactsRepository, None, None]:  # pylint:
         ArtifactsRepository: The artifacts repository for the specific test
         case.
     """
-    artifacts_dir_for_test_case = (
-        Path(request.config.artifacts_dir).resolve() / request.node.name
-    )
-    with ArtifactsRepository(artifacts_dir_for_test_case) as repo:
+    base_artifacts_dir = Path(request.config.artifacts_dir).resolve()
+
+    if request.config.artifacts_use_subdir_for_parametrize:
+        try:
+            test_case_artifacts_dir = (
+                base_artifacts_dir
+                / request.node.originalname
+                / request.node.callspec.id
+            )
+        except AttributeError:
+            test_case_artifacts_dir = base_artifacts_dir / request.node.originalname
+    else:
+        test_case_artifacts_dir = base_artifacts_dir / request.node.originalname
+
+    with ArtifactsRepository(test_case_artifacts_dir) as repo:
         yield repo
 
 

tests/test_fixtures.py

@@ -31,8 +31,12 @@
         ),
     ],
 )
-def test_fixture_artifacts_dir(pytester, addopts, ini, expected):
-    """Test that the artifacts_dir fixture returns the default value."""
+def test_can_configure_artifacts_dir(pytester, addopts, ini, expected):
+    """Test that the artifacts_dir fixture can be configured from
+    * pytest ini file
+    * command line arguments
+    * both ini and command line arguments, with command line taking precedence
+    """
     pytester.makeini(ini)
 
     # create a temporary pytest test module
@@ -55,6 +59,41 @@ def test_sth(request):
     assert result.ret == 0
 
 
+def test_can_configure_artifacts_use_subdir_for_parametrize(pytester):
+    """Test that the artifacts_use_subdir_for_parametrize fixture can be configured from
+    pytest ini file.
+    """
+    pytester.makeini("""
+        [pytest]
+        artifacts_use_subdir_for_parametrize = true
+        """)
+
+    # create a temporary pytest test module
+    pytester.makepyfile("""
+        import pytest
+
+        @pytest.mark.parametrize("text", ["hello", "goodbye"])
+        def test_sth(text, request, artifacts):
+            assert request.config.artifacts_use_subdir_for_parametrize is True
+
+            with artifacts.open("foobar.txt", mode="wt") as f:
+                f.write(text)
+                f.flush()
+
+            with artifacts.open("foobar.txt", mode="rt") as f:
+                assert f.readlines() == [text]
+
+            assert artifacts.dir.is_dir() and artifacts.dir.name in ["hello", "goodbye"]
+
+    """)
+
+    # run pytest with the following cmd args
+    result = pytester.runpytest("-v")
+
+    # make sure that we get a '0' exit code for the testsuite
+    assert result.ret == 0
+
+
 def test_fixture_artifacts(pytester):
     """Test that writing to the artifacts creates a directory named after the
     test case.