From 24afc0931bc25205ae033a7b60b00165a384c00b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20Tr=C3=B6ger?= Date: Sat, 20 Jan 2024 18:18:01 +1000 Subject: [PATCH] =?UTF-8?q?feat:=20run=20doctest=20as=20part=20of=20runnin?= =?UTF-8?q?g=20tests,=20which=20collects=20doctests=20from=20both=20the=20?= =?UTF-8?q?package=E2=80=99s=20doc=20strings=20and=20the=20package=20docum?= =?UTF-8?q?entation=20(#637)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .pre-commit-config.yaml | 2 +- README.md | 35 +++++++++++++++++++---------------- docs/source/index.rst | 12 ++++++++++++ pyproject.toml | 24 ++++++++++++++++++++---- src/package/something.py | 14 +++++++++++++- 5 files changed, 65 insertions(+), 22 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 24d4fab2..e3837f2c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -162,7 +162,7 @@ repos: hooks: - id: pytest name: Run unit tests - entry: pytest -c pyproject.toml --cov-config pyproject.toml + entry: pytest -c pyproject.toml --cov-config pyproject.toml src/package/ tests/ docs/ language: python verbose: true always_run: true diff --git a/README.md b/README.md index 394b45c1..09174512 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ A number of git hooks are invoked before and after a commit, and before push. Th ### Unit testing -Comprehensive unit testing is enabled using [pytest](https://pytest.org/) combined with [Hypothesis](https://hypothesis.works/) (to generate test payloads and strategies), and test code and branch coverage is measured using [coverage](https://github.com/nedbat/coveragepy) (see [below](#testing)). +Comprehensive unit testing is enabled using [pytest](https://pytest.org/) combined with [doctest](https://docs.python.org/3/library/doctest.html) and [Hypothesis](https://hypothesis.works/) (to support [property-based testing](https://en.wikipedia.org/wiki/Software_testing#Property_testing)), and both code and branch coverage are measured using [coverage](https://github.com/nedbat/coveragepy) (see [below](#testing)). ### Documentation @@ -165,25 +165,28 @@ As mentioned above, this repository is set up to use [pytest](https://pytest.org ```bash make test ``` -which runs all tests in both your local Python virtual environment. For more options, see the [pytest command-line flags](https://docs.pytest.org/en/6.2.x/reference.html#command-line-flags). Also note that pytest includes [doctest](https://docs.python.org/3/library/doctest.html), which means that module and function [docstrings](https://www.python.org/dev/peps/pep-0257/#what-is-a-docstring) may contain test code that executes as part of the unit tests. +which runs all tests in both your local Python virtual environment. For more options, see the [pytest command-line flags](https://docs.pytest.org/en/7.4.x/reference/reference.html#command-line-flags). Also note that pytest includes [doctest](https://docs.python.org/3/library/doctest.html), which means that module and function [docstrings](https://www.python.org/dev/peps/pep-0257/#what-is-a-docstring), as well as the documentation, may contain test code that executes as part of the unit tests. -Test code and branch coverage is already tracked using [coverage](https://github.com/nedbat/coveragepy) and the [pytest-cov](https://github.com/pytest-dev/pytest-cov) plugin for pytest, and it measures how much code in the `src/package/` folder is covered by tests: +Both statement and branch coverage are being tracked using [coverage](https://github.com/nedbat/coveragepy) and the [pytest-cov](https://github.com/pytest-dev/pytest-cov) plugin for pytest, and it measures how much code in the `src/package/` folder is covered by tests: ``` Run unit tests...........................................................Passed - hook id: pytest -- duration: 0.48s +- duration: 0.6s ============================= test session starts ============================== -platform darwin -- Python 3.10.2, pytest-6.2.5, py-1.11.0, pluggy-1.0.0 -- /.../python-package-template/.venv/bin/python3.10 +platform darwin -- Python 3.11.7, pytest-7.4.4, pluggy-1.3.0 -- /path/to/python-package-template/.venv/bin/python cachedir: .pytest_cache -hypothesis profile 'default' -> database=DirectoryBasedExampleDatabase('/.../python-package-template/.hypothesis/examples') -rootdir: /.../python-package-template, configfile: pyproject.toml, testpaths: tests -plugins: hypothesis-6.41.0, cov-3.0.0 -collected 1 item +hypothesis profile 'default-with-verbose-verbosity-with-explain-phase' -> max_examples=500, verbosity=Verbosity.verbose, phases=(Phase.explicit, Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain), database=DirectoryBasedExampleDatabase('/path/to/python-package-template/.hypothesis/examples') +rootdir: /path/to/python-package-template +configfile: pyproject.toml +plugins: custom-exit-code-0.3.0, cov-4.1.0, doctestplus-1.1.0, hypothesis-6.90.0, env-1.1.1 +collected 3 items -tests/test_something.py::test_something PASSED [100%] +src/package/something.py::package.something.Something.do_something PASSED [ 33%] +tests/test_something.py::test_something PASSED [ 66%] +docs/source/index.rst::index.rst PASSED [100%] ----------- coverage: platform darwin, python 3.10.2-final-0 ---------- +---------- coverage: platform darwin, python 3.11.7-final-0 ---------- Name Stmts Miss Branch BrPart Cover Missing ---------------------------------------------------------------------- src/package/__init__.py 1 0 0 0 100% @@ -197,20 +200,20 @@ Required test coverage of 100.0% reached. Total coverage: 100.00% tests/test_something.py::test_something: - during reuse phase (0.00 seconds): - - Typical runtimes: ~ 1ms, ~ 28% in data generation + - Typical runtimes: < 1ms, of which < 1ms in data generation - 1 passing examples, 0 failing examples, 0 invalid examples - during generate phase (0.00 seconds): - - Typical runtimes: < 1ms, ~ 43% in data generation + - Typical runtimes: < 1ms, of which < 1ms in data generation - 1 passing examples, 0 failing examples, 0 invalid examples - Stopped because nothing left to do -============================== 1 passed in 0.16s =============================== +============================== 3 passed in 0.05s =============================== ``` -Note that code that’s not covered by tests is listed under the `Missing` column, and branches not taken too. The net effect of enforcing 100% code and branch coverage is that every new major and minor feature, every code change, and every fix are being tested (keeping in mind that high _coverage_ does not necessarily imply comprehensive _test data_). +Note that code that’s not covered by tests is listed under the `Missing` column, and branches not taken too. The net effect of enforcing 100% code and branch coverage is that every new major and minor feature, every code change, and every fix are being tested (keeping in mind that high _coverage_ does not imply comprehensive, meaningful _test data_). -Hypothesis is a package that implements [property based testing](https://en.wikipedia.org/wiki/QuickCheck) and that provides payload generation for your tests based on strategy descriptions ([more](https://hypothesis.works/#what-is-hypothesis)). Using its [pytest plugin](https://hypothesis.readthedocs.io/en/latest/details.html#the-hypothesis-pytest-plugin) Hypothesis is ready to be used for this package. +Hypothesis is a package that implements [property based testing](https://en.wikipedia.org/wiki/Software_testing#Property_testing) and that provides payload generation for your tests based on strategy descriptions ([more](https://hypothesis.works/#what-is-hypothesis)). Using its [pytest plugin](https://hypothesis.readthedocs.io/en/latest/details.html#the-hypothesis-pytest-plugin) Hypothesis is ready to be used for this package. ## Generating documentation diff --git a/docs/source/index.rst b/docs/source/index.rst index da8e23a6..a4c5f1b1 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -16,6 +16,18 @@ Package package Something ========= +The ``Something`` module contains a useful class which allows you to do something +like the following: + +.. code: pycon + + >>> from package import something + >>> s = something.Something() + >>> s.do_something() + True + >>> s.do_something(False) # doctest: +SKIP + False # This value would fail the test. + .. automodule:: package :members: diff --git a/pyproject.toml b/pyproject.toml index 45fa6c69..48079532 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ test = [ "pytest >=7.2.0,<8.0.0", "pytest-custom_exit_code ==0.3.0", "pytest-cov ==4.1.0", + "pytest-doctestplus ==1.1.0", "pytest-env ==1.1.1", ] @@ -207,13 +208,28 @@ max-line-length = 120 # https://docs.pytest.org/en/latest/reference/customize.html#configuration-file-formats # https://docs.pytest.org/en/latest/reference/reference.html#configuration-options # https://docs.pytest.org/en/latest/reference/reference.html#command-line-flags +# +# To integrate Hypothesis into pytest and coverage, we use its native plugin: +# https://hypothesis.readthedocs.io/en/latest/details.html#the-hypothesis-pytest-plugin +# +# To discover tests in documentation, we use doctest and the doctest-plus plugin which +# adds multiple useful options to control tests in documentation. More details at: +# https://docs.python.org/3/library/doctest.html +# https://github.com/scientific-python/pytest-doctestplus +# +# To avoid failing pytest when no tests were dicovered, we need an extra plugin: +# https://docs.pytest.org/en/latest/reference/exit-codes.html +# https://github.com/yashtodi94/pytest-custom_exit_code [tool.pytest.ini_options] minversion = "7.0" -addopts = "-vv --doctest-modules --tb native --hypothesis-show-statistics --hypothesis-explain --hypothesis-verbosity verbose -ra --cov package" # Consider adding --pdb +addopts = """-vv -ra --tb native \ + --hypothesis-show-statistics --hypothesis-explain --hypothesis-verbosity verbose \ + --doctest-modules --doctest-continue-on-failure --doctest-glob '*.rst' --doctest-plus \ + --suppress-no-test-exit-code \ + --cov package \ +""" # Consider adding --pdb +# https://docs.python.org/3/library/doctest.html#option-flags doctest_optionflags = "IGNORE_EXCEPTION_DETAIL" -testpaths = [ - "tests", -] env = [ "PYTHONDEVMODE=1", # https://docs.python.org/3/library/devmode.html ] diff --git a/src/package/something.py b/src/package/something.py index c89fcf2a..17adf42c 100644 --- a/src/package/something.py +++ b/src/package/something.py @@ -6,5 +6,17 @@ class Something: @staticmethod def do_something(value: bool = False) -> bool: - """Return true, always.""" + """Return true, always. + + Test this function in your local terminal, too, for example: + + .. code: pycon + + >>> s = Something() + >>> s.do_something(False) + True + >>> s.do_something(value=True) + True + + """ return value or True