diff --git a/.github/workflows/_build.yaml b/.github/workflows/_build.yaml index e43391a2..93168bce 100644 --- a/.github/workflows/_build.yaml +++ b/.github/workflows/_build.yaml @@ -79,9 +79,13 @@ jobs: # Build the sdist and wheel distribution of the package and docs as a zip file. # We don't need to check and test the package separately because `make dist` runs - # those targets first and only builds the package if they succeed. + # those targets first and only builds the package if they succeed. The Hypothesis + # profile picks the Hypothesis settings (see tests/conftest.py) which, for Github, + # disable randomized testing to avoid breaking CI. - name: Build the package run: make dist + env: + HYPOTHESIS_PROFILE: github # Generate the requirements.txt that contains the hash digests of the dependencies and # generate the SBOM using CyclonDX SBOM generator. diff --git a/README.md b/README.md index a70e3e45..ef46743a 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,19 @@ src/package/something.py 4 0 100% TOTAL 5 0 100% Required test coverage of 100.0% reached. Total coverage: 100.00% +============================ Hypothesis Statistics ============================= + +tests/test_something.py::test_something: + + - during reuse phase (0.00 seconds): + - Typical runtimes: ~ 1ms, ~ 28% in data generation + - 1 passing examples, 0 failing examples, 0 invalid examples + + - during generate phase (0.00 seconds): + - Typical runtimes: < 1ms, ~ 43% in data generation + - 1 passing examples, 0 failing examples, 0 invalid examples + + - Stopped because nothing left to do ============================== 1 passed in 0.16s =============================== ``` diff --git a/pyproject.toml b/pyproject.toml index 264d209d..c53256e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -199,7 +199,7 @@ max-line-length = 120 # https://docs.pytest.org/en/latest/reference/reference.html#command-line-flags [tool.pytest.ini_options] minversion = "7.0" -addopts = "--verbose --doctest-modules -ra --cov package" # Consider adding --pdb +addopts = "--verbose --doctest-modules --hypothesis-show-statistics --hypothesis-explain --hypothesis-verbosity verbose -ra --cov package" # Consider adding --pdb doctest_optionflags = "IGNORE_EXCEPTION_DETAIL" testpaths = [ "tests", diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..a3bb8d10 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,13 @@ +"""Test configuration and other goodness.""" + +import os + +import hypothesis + +# Configure Hypothesis. For Github CI we derandomize to prevent nondeterministic tests +# because we don't want publishing to fail randomly. However, targeted fuzzing should +# use its own profile and randomize. +hypothesis.settings.register_profile("default", max_examples=500, derandomize=False) +hypothesis.settings.register_profile("github", max_examples=100, derandomize=True) +hypothesis.settings.register_profile("fuzz", max_examples=10000, derandomize=False) +hypothesis.settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default"))