chore(test): add Hypothesis command-line switches and configuration p…

…rofiles (#437)

.github/workflows/_build.yaml

@@ -79,9 +79,13 @@ jobs:
 
     # Build the sdist and wheel distribution of the package and docs as a zip file.
     # We don't need to check and test the package separately because `make dist` runs
-    # those targets first and only builds the package if they succeed.
+    # those targets first and only builds the package if they succeed. The Hypothesis
+    # profile picks the Hypothesis settings (see tests/conftest.py) which, for Github,
+    # disable randomized testing to avoid breaking CI.
     - name: Build the package
       run: make dist
+      env:
+        HYPOTHESIS_PROFILE: github
 
     # Generate the requirements.txt that contains the hash digests of the dependencies and
     # generate the SBOM using CyclonDX SBOM generator.

README.md

@@ -190,6 +190,19 @@ src/package/something.py       4      0   100%
 TOTAL                          5      0   100%
 
 Required test coverage of 100.0% reached. Total coverage: 100.00%
+============================ Hypothesis Statistics =============================
+
+tests/test_something.py::test_something:
+
+  - during reuse phase (0.00 seconds):
+    - Typical runtimes: ~ 1ms, ~ 28% in data generation
+    - 1 passing examples, 0 failing examples, 0 invalid examples
+
+  - during generate phase (0.00 seconds):
+    - Typical runtimes: < 1ms, ~ 43% in data generation
+    - 1 passing examples, 0 failing examples, 0 invalid examples
+
+  - Stopped because nothing left to do
 
 ============================== 1 passed in 0.16s ===============================
 ```

pyproject.toml

@@ -199,7 +199,7 @@ max-line-length = 120
 # https://docs.pytest.org/en/latest/reference/reference.html#command-line-flags
 [tool.pytest.ini_options]
 minversion = "7.0"
-addopts = "--verbose --doctest-modules -ra --cov package"  # Consider adding --pdb
+addopts = "--verbose --doctest-modules --hypothesis-show-statistics --hypothesis-explain --hypothesis-verbosity verbose -ra --cov package"  # Consider adding --pdb
 doctest_optionflags = "IGNORE_EXCEPTION_DETAIL"
 testpaths = [
     "tests",

tests/conftest.py

@@ -0,0 +1,13 @@
+"""Test configuration and other goodness."""
+
+import os
+
+import hypothesis
+
+# Configure Hypothesis. For Github CI we derandomize to prevent nondeterministic tests
+# because we don't want publishing to fail randomly. However, targeted fuzzing should
+# use its own profile and randomize.
+hypothesis.settings.register_profile("default", max_examples=500, derandomize=False)
+hypothesis.settings.register_profile("github", max_examples=100, derandomize=True)
+hypothesis.settings.register_profile("fuzz", max_examples=10000, derandomize=False)
+hypothesis.settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default"))