test: Add tests highlighting xdist incompatibilities

pyproject.toml

@@ -95,6 +95,7 @@ dist,
 [tool.pytest.ini_options]
 addopts = '-p syrupy -p pytester -p no:legacypath --doctest-modules'
 testpaths = ['tests']
+xfail_strict = true
 
 [tool.coverage.run]
 source = ['./src']

tests/conftest.py

@@ -0,0 +1,33 @@
+from typing import List
+
+import pytest
+
+# Constants for testing with extra plugin arguments
+_NO_ARGS = []
+_XDIST_ZERO = ["--numprocesses", "0"]
+_XDIST_TWO = ["--numprocesses", "2"]
+
+
+@pytest.fixture(
+    params=[_NO_ARGS, _XDIST_ZERO, _XDIST_TWO],
+    ids=["no_plugin", "xdist_zero", "xdist_two"],
+)
+def plugin_args(request: pytest.FixtureRequest) -> List[str]:
+    """Fixture to test with various plugins"""
+    return request.param
+
+
+@pytest.fixture(
+    params=[
+        _NO_ARGS,
+        _XDIST_ZERO,
+        pytest.param(
+            _XDIST_TWO,
+            marks=pytest.mark.xfail(reason="Not currently compatible with xdist"),
+        ),
+    ],
+    ids=["no_plugin", "xdist_zero", "xdist_two"],
+)
+def plugin_args_fails_xdist(request: pytest.FixtureRequest) -> List[str]:
+    """Fixture to test with various plugins, but expected to fail xdist"""
+    return request.param

tests/integration/test_custom_comparator.py

@@ -48,29 +48,27 @@ def generate_snapshots(testdir, testcases_initial):
     return result, testdir, testcases_initial
 
 
-@pytest.mark.xfail(strict=False)
 def test_generated_snapshots(generate_snapshots):
     result = generate_snapshots[0]
     result.stdout.re_match_lines((r"1 snapshot generated\."))
     assert "snapshots unused" not in result.stdout.str()
     assert result.ret == 0
 
 
-@pytest.mark.xfail(strict=False)
-def test_approximate_match(generate_snapshots):
+def test_approximate_match(generate_snapshots, plugin_args_fails_xdist):
     testdir = generate_snapshots[1]
     testdir.makepyfile(
         test_file="""
             def test_passed_custom(snapshot_custom):
                 assert snapshot_custom == 3.2
             """
     )
-    result = testdir.runpytest("-v")
+    result = testdir.runpytest("-v", *plugin_args_fails_xdist)
     result.stdout.re_match_lines((r"test_file.py::test_passed_custom PASSED"))
     assert result.ret == 0
 
 
-@pytest.mark.xfail(strict=False)
+# xdist behavior different on Python 3.8 - needs to be reviewed
 def test_failed_snapshots(generate_snapshots):
     testdir = generate_snapshots[1]
     testdir.makepyfile(test_file=generate_snapshots[2]["failed"])
@@ -79,10 +77,9 @@ def test_failed_snapshots(generate_snapshots):
     assert result.ret == 1
 
 
-@pytest.mark.xfail(strict=False)
-def test_updated_snapshots(generate_snapshots):
+def test_updated_snapshots(generate_snapshots, plugin_args_fails_xdist):
     _, testdir, initial = generate_snapshots
     testdir.makepyfile(test_file=initial["failed"])
-    result = testdir.runpytest("-v", "--snapshot-update")
+    result = testdir.runpytest("-v", "--snapshot-update", *plugin_args_fails_xdist)
     result.stdout.re_match_lines((r"1 snapshot updated\."))
     assert result.ret == 0

tests/integration/test_pycharm_patch.py

@@ -23,35 +23,37 @@ def __init__(self, expected, actual, msg=None, preformated=False, real_exception
 
 
 @pytest.mark.filterwarnings("default")
-def test_logs_a_warning_if_unable_to_apply_patch(testdir):
+def test_logs_a_warning_if_unable_to_apply_patch(testdir, plugin_args):
     testdir.makepyfile(
         test_file="""
     def test_case(snapshot):
         assert snapshot == [1, 2]
     """
     )
-    testdir.runpytest("-v", "--snapshot-update")
+    testdir.runpytest("-v", "--snapshot-update", *plugin_args)
     testdir.makepyfile(
         test_file="""
     def test_case(snapshot):
         assert snapshot == [1, 2, 3]
     """
     )
 
-    result = testdir.runpytest("-v", "--snapshot-patch-pycharm-diff")
+    result = testdir.runpytest("-v", "--snapshot-patch-pycharm-diff", *plugin_args)
     result.assert_outcomes(failed=1, passed=0, warnings=1)
 
 
 @pytest.mark.filterwarnings("default")
-def test_patches_pycharm_diff_tools_when_flag_set(testdir, mock_teamcity_diff_tools):
+def test_patches_pycharm_diff_tools_when_flag_set(
+    testdir, mock_teamcity_diff_tools, plugin_args
+):
     # Generate initial snapshot
     testdir.makepyfile(
         test_file="""
     def test_case(snapshot):
         assert snapshot == [1, 2]
     """
     )
-    testdir.runpytest("-v", "--snapshot-update")
+    testdir.runpytest("-v", "--snapshot-update", *plugin_args)
 
     # Generate diff and mimic EqualsAssertionError being thrown
     testdir.makepyfile(
@@ -70,7 +72,7 @@ def test_case(snapshot):
     """
     )
 
-    result = testdir.runpytest("-v", "--snapshot-patch-pycharm-diff")
+    result = testdir.runpytest("-v", "--snapshot-patch-pycharm-diff", *plugin_args)
     # No warnings because patch should have been successful
     result.assert_outcomes(failed=1, passed=0, warnings=0)
 
@@ -85,7 +87,7 @@ def test_case(snapshot):
 
 @pytest.mark.filterwarnings("default")
 def test_patches_pycharm_diff_tools_when_flag_set_and_snapshot_on_right(
-    testdir, mock_teamcity_diff_tools
+    testdir, mock_teamcity_diff_tools, plugin_args
 ):
     # Generate initial snapshot
     testdir.makepyfile(
@@ -94,7 +96,7 @@ def test_case(snapshot):
         assert [1, 2] == snapshot
     """
     )
-    testdir.runpytest("-v", "--snapshot-update")
+    testdir.runpytest("-v", "--snapshot-update", *plugin_args)
 
     # Generate diff and mimic EqualsAssertionError being thrown
     testdir.makepyfile(
@@ -113,7 +115,7 @@ def test_case(snapshot):
     """
     )
 
-    result = testdir.runpytest("-v", "--snapshot-patch-pycharm-diff")
+    result = testdir.runpytest("-v", "--snapshot-patch-pycharm-diff", *plugin_args)
     # No warnings because patch should have been successful
     result.assert_outcomes(failed=1, passed=0, warnings=0)
 
@@ -128,7 +130,7 @@ def test_case(snapshot):
 
 @pytest.mark.filterwarnings("default")
 def test_it_does_not_patch_pycharm_diff_tools_by_default(
-    testdir, mock_teamcity_diff_tools
+    testdir, mock_teamcity_diff_tools, plugin_args
 ):
     # Generate initial snapshot
     testdir.makepyfile(
@@ -137,7 +139,7 @@ def test_case(snapshot):
         assert snapshot == [1, 2]
     """
     )
-    testdir.runpytest("-v", "--snapshot-update")
+    testdir.runpytest("-v", "--snapshot-update", *plugin_args)
 
     # Generate diff and mimic EqualsAssertionError being thrown
     testdir.makepyfile(
@@ -156,7 +158,7 @@ def test_case(snapshot):
     """
     )
 
-    result = testdir.runpytest("-v")
+    result = testdir.runpytest("-v", *plugin_args)
     # No warnings because patch should have been successful
     result.assert_outcomes(failed=1, passed=0, warnings=0)
 
@@ -170,7 +172,9 @@ def test_case(snapshot):
 
 
 @pytest.mark.filterwarnings("default")
-def test_it_has_no_impact_on_non_syrupy_assertions(testdir, mock_teamcity_diff_tools):
+def test_it_has_no_impact_on_non_syrupy_assertions(
+    testdir, mock_teamcity_diff_tools, plugin_args
+):
     # Generate diff and mimic EqualsAssertionError being thrown
     testdir.makepyfile(
         test_file="""
@@ -188,7 +192,7 @@ def test_case():
     """
     )
 
-    result = testdir.runpytest("-v")
+    result = testdir.runpytest("-v", *plugin_args)
     # No warnings because patch should have been successful
     result.assert_outcomes(failed=1, passed=0, warnings=0)
 
@@ -202,7 +206,7 @@ def test_case():
 
 @pytest.mark.filterwarnings("default")
 def test_has_no_impact_on_real_exceptions_that_are_not_assertion_errors(
-    testdir, mock_teamcity_diff_tools
+    testdir, mock_teamcity_diff_tools, plugin_args
 ):
     # Generate diff and mimic EqualsAssertionError being thrown
     testdir.makepyfile(
@@ -225,7 +229,7 @@ def test_case():
     """
     )
 
-    result = testdir.runpytest("-v")
+    result = testdir.runpytest("-v", *plugin_args)
     # No warnings because patch should have been successful
     result.assert_outcomes(failed=1, passed=0, warnings=0)
 

tests/integration/test_pytest_extension.py

@@ -1,4 +1,4 @@
-def test_ignores_non_function_nodes(testdir):
+def test_ignores_non_function_nodes(testdir, plugin_args):
     conftest = """
         import pytest
 
@@ -23,25 +23,30 @@ def test_example(snapshot):
         """
     testdir.makeconftest(conftest)
     testdir.makepyfile(test_file=testcase)
-    result = testdir.runpytest("test_file.py", "-v", "--snapshot-update")
+    result = testdir.runpytest("test_file.py", "-v", "--snapshot-update", *plugin_args)
     result.stdout.re_match_lines((r".*test_file.py::CUSTOM.*"))
     assert result.ret == 0
 
 
-def test_handles_pyargs_non_module_when_both_given(testdir):
+def test_handles_pyargs_non_module_when_both_given(testdir, plugin_args):
     testdir.makeconftest("")
     testcase = """
         def test_example(snapshot):
             assert snapshot == 1
         """
     testdir.makepyfile(test_file=testcase)
     result = testdir.runpytest(
-        "-v", "test_file.py", "--pyargs", "test_file", "--snapshot-update"
+        "-v",
+        "test_file.py",
+        "--pyargs",
+        "test_file",
+        "--snapshot-update",
+        *plugin_args,
     )
     assert result.ret == 0
 
 
-def test_does_not_print_empty_snapshot_report(testdir):
+def test_does_not_print_empty_snapshot_report(testdir, plugin_args_fails_xdist):
     testdir.makeconftest("")
     testcase_no_snapshots = """
         def test_example(snapshot):
@@ -55,12 +60,16 @@ def test_example(snapshot):
         test_file_no=testcase_no_snapshots, test_file_yes=testcase_yes_snapshots
     )
 
-    result = testdir.runpytest("-v", "test_file_no.py", "--snapshot-update")
+    result = testdir.runpytest(
+        "-v", "test_file_no.py", "--snapshot-update", *plugin_args_fails_xdist
+    )
     result.stdout.re_match_lines((r".*test_file_no.py.*"))
     assert "snapshot report" not in result.stdout.str()
     assert "test_file_yes" not in result.stdout.str()
     assert result.ret == 0
 
-    result = testdir.runpytest("-v", "test_file_yes.py", "--snapshot-update")
+    result = testdir.runpytest(
+        "-v", "test_file_yes.py", "--snapshot-update", *plugin_args_fails_xdist
+    )
     result.stdout.re_match_lines((r".*test_file_yes.py.*", r".*snapshot report.*"))
     assert result.ret == 0

tests/integration/test_single_file_multiple_extensions.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 
 
-def test_multiple_file_extensions(testdir):
+def test_multiple_file_extensions(testdir, plugin_args_fails_xdist):
     file_extension = "ext2.ext1"
 
     testcase = f"""
@@ -21,7 +21,7 @@ def test_dot_in_filename(snapshot):
 
     test_file: Path = testdir.makepyfile(test_file=testcase)
 
-    result = testdir.runpytest("-v", "--snapshot-update")
+    result = testdir.runpytest("-v", "--snapshot-update", *plugin_args_fails_xdist)
     result.stdout.re_match_lines((r"1 snapshot generated\."))
     assert "snapshots unused" not in result.stdout.str()
     assert result.ret == 0
@@ -34,13 +34,13 @@ def test_dot_in_filename(snapshot):
     )
     assert snapshot_file.exists()
 
-    result = testdir.runpytest("-v")
+    result = testdir.runpytest("-v", *plugin_args_fails_xdist)
     result.stdout.re_match_lines((r"1 snapshot passed\."))
     assert "snapshots unused" not in result.stdout.str()
     assert result.ret == 0
 
 
-def test_class_style(testdir):
+def test_class_style(testdir, plugin_args_fails_xdist):
     """
     Regression test for https://github.com/syrupy-project/syrupy/issues/717
     """
@@ -60,7 +60,7 @@ def test_foo(self, snapshot):
 
     test_file: Path = testdir.makepyfile(test_file=testcase)
 
-    result = testdir.runpytest("-v", "--snapshot-update")
+    result = testdir.runpytest("-v", "--snapshot-update", *plugin_args_fails_xdist)
     result.stdout.re_match_lines((r"1 snapshot generated\."))
     assert "deleted" not in result.stdout.str()
     assert result.ret == 0
@@ -70,7 +70,7 @@ def test_foo(self, snapshot):
     )
     assert snapshot_file.exists()
 
-    result = testdir.runpytest("-v")
+    result = testdir.runpytest("-v", *plugin_args_fails_xdist)
     result.stdout.re_match_lines((r"1 snapshot passed\."))
     assert "snapshots unused" not in result.stdout.str()
     assert result.ret == 0