Prune patchwork Sphinx config; add class inheritance diagrams

.github/workflows/github-pages.yml

@@ -43,6 +43,11 @@ jobs:
         with:
           python-version-file: ".python-version"
 
+      # Graphviz (`dot`) is required by sphinx.ext.inheritance_diagram for the
+      # class diagrams on the API page.
+      - name: Install Graphviz
+        run: sudo apt-get update && sudo apt-get install -y graphviz
+
       - name: Install docs dependencies
         run: |
           uv sync --group docs

.github/workflows/validate.yaml

@@ -51,6 +51,11 @@ jobs:
         with:
           python-version-file: ".python-version"
 
+      # Graphviz (`dot`) is required by sphinx.ext.inheritance_diagram for the
+      # class diagrams on the API page.
+      - name: Install Graphviz
+        run: sudo apt-get update && sudo apt-get install -y graphviz
+
       - name: Install docs dependencies
         run: |
           uv sync --group docs

DEVELOPMENT.md

@@ -168,6 +168,15 @@ dependency group; `uv sync --all-groups` installs them too):
 uv sync --group docs
 ```
 
+The class diagrams on the API page are rendered by
+[`sphinx.ext.inheritance_diagram`](https://www.sphinx-doc.org/en/master/usage/extensions/inheritance.html),
+which needs [Graphviz](https://graphviz.org/) (the `dot` binary) on your `PATH`:
+
+```bash
+brew install graphviz        # macOS
+sudo apt-get install graphviz # Debian/Ubuntu
+```
+
 Build the site once:
 
 ```bash
@@ -200,17 +209,32 @@ make docclean docautobuild
 ```
 
 Sphinx prints `WARNING:` lines during the build and ends with a summary
-(e.g. `build succeeded, N warnings.`). Warnings are worth scanning — they flag broken
-cross-references, malformed docstrings, and members that failed to render.
+(e.g. `build succeeded, N warnings.`). The CI docs build runs with
+`SPHINXOPTS="-W --keep-going"`, which turns warnings into errors (and reports all of
+them before failing), so the build must be warning-free to pass. Reproduce that
+locally with:
+
+```bash
+make docclean
+make html SPHINXOPTS="-W --keep-going"
+```
+
+You can also check for broken links and cross-references (this makes network requests,
+so it's not part of CI):
+
+```bash
+make linkcheck
+```
 
 A note on the API reference: FlexEval's database models (`src/flexeval/classes/`) are
 [peewee](https://docs.peewee-orm.com/) models, and peewee's metaclass rewrites
 field definitions into descriptors and adds generated members (a per-model
-`DoesNotExist` exception and a `<fk>_id` alias for every foreign key). The
-`skip_peewee_internals` hook in `docs/conf.py` hides that generated noise while
-keeping the real fields, and `inherited-members: False` keeps inherited peewee/pydantic
-machinery out of the reference. If model attributes stop appearing on a generated page,
-that hook (and the `autodoc_default_options`) is the place to look.
+`DoesNotExist` exception, a `<fk>_id` alias for every foreign key, and a
+back-reference accessor for every incoming relation). The `skip_peewee_internals` hook
+in `docs/conf.py` hides that generated noise while keeping the real fields, and
+`inherited-members: False` keeps inherited peewee/pydantic machinery out of the
+reference. If model attributes stop appearing on a generated page, that hook (and the
+`autodoc_default_options`) is the place to look.
 
 ## Releasing a new version
 

docs/api.rst

@@ -5,10 +5,39 @@ The API for FlexEval is still largely undocumented, although the package is not
 
 A good place to start is with the Pydantic class :class:`~flexeval.schema.evalrun_schema.EvalRun`, which defines the inputs expected by FlexEval.
 
-Packages:
+Data sources
+------------
+
+FlexEval accepts several kinds of data source, all sharing a common
+:class:`~flexeval.schema.evalrun_schema.DataSource` base:
+
+.. inheritance-diagram:: flexeval.schema.evalrun_schema.FileDataSource
+                         flexeval.schema.evalrun_schema.NamedDataSource
+                         flexeval.schema.evalrun_schema.IterableDataSource
+   :parts: 1
+   :top-classes: flexeval.schema.evalrun_schema.DataSource
+
+Database models
+---------------
+
+Loaded data is stored in a hierarchy of `peewee <https://docs.peewee-orm.com/>`_
+models, all descending from a common base:
+
+.. inheritance-diagram:: flexeval.classes.dataset.Dataset
+                         flexeval.classes.thread.Thread
+                         flexeval.classes.turn.Turn
+                         flexeval.classes.message.Message
+                         flexeval.classes.tool_call.ToolCall
+                         flexeval.classes.metric.Metric
+                         flexeval.classes.eval_set_run.EvalSetRun
+   :parts: 1
+   :top-classes: flexeval.classes.base.BaseModel
+
+Packages
+--------
 
 .. autosummary::
    :toctree: generated
    :recursive:
 
-   flexeval
+   flexeval

docs/conf.py

@@ -4,14 +4,9 @@
 # https://www.sphinx-doc.org/en/master/usage/configuration.html
 import sys
 import os
-import inspect
-from pathlib import Path
-from packaging.version import parse as parse_version
 
 import peewee as pw
 
-import flexeval
-
 sys.path.append(os.path.abspath("."))
 sys.path.append(".")
 
@@ -30,24 +25,15 @@
 extensions = [
     "sphinx.ext.autodoc",
     "sphinx.ext.autosummary",
-    "sphinx.ext.inheritance_diagram",
+    "sphinx.ext.inheritance_diagram",  # class diagrams on the API page (needs Graphviz)
     "sphinx.ext.intersphinx",
-    # "IPython.sphinxext.ipython_console_highlighting",
-    # "IPython.sphinxext.ipython_directive",
-    "numpydoc",  # Needs to be loaded *after* autodoc.
-    "sphinx.ext.napoleon",
-    "matplotlib.sphinxext.plot_directive",
-    "matplotlib.sphinxext.roles",
-    "matplotlib.sphinxext.figmpl_directive",
+    "sphinx.ext.napoleon",  # Google-style docstrings
     "sphinxext.github",
     "sphinx_copybutton",
-    "sphinx_design",
-    "sphinx_tags",
-    "sphinx.ext.linkcode",
     # "myst_parser",  # don't use myst_parser with myst_nb; it is automatically loaded by myst_parser
     "myst_nb",
     "sphinxcontrib.autodoc_pydantic",  # because sphinx plays badly with pydantic
-    "sphinx.ext.viewcode",  # should add a view code link
+    "sphinx.ext.viewcode",  # adds a "[source]" link next to documented objects
     "sphinxcontrib.programoutput",  # for inline bash execution
 ]
 source_suffix = {
@@ -91,67 +77,6 @@
     "pydantic": ("https://docs.pydantic.dev/latest", None),
 }
 
-docutils_conf = {
-    "line-length-limit": None,  # Disable docutils line-length limit
-}
-
-
-def linkcode_resolve(domain, info):
-    """
-    Determine the URL corresponding to Python object
-    """
-    if domain != "py":
-        return None
-
-    modname = info["module"]
-    fullname = info["fullname"]
-
-    submod = sys.modules.get(modname)
-    if submod is None:
-        return None
-
-    obj = submod
-    for part in fullname.split("."):
-        try:
-            obj = getattr(obj, part)
-        except AttributeError:
-            return None
-
-    if inspect.isfunction(obj):
-        obj = inspect.unwrap(obj)
-    try:
-        fn = inspect.getsourcefile(obj)
-    except TypeError:
-        fn = None
-    if not fn or fn.endswith("__init__.py"):
-        try:
-            fn = inspect.getsourcefile(sys.modules[obj.__module__])
-        except (TypeError, AttributeError, KeyError):
-            fn = None
-    if not fn:
-        return None
-
-    try:
-        source, lineno = inspect.getsourcelines(obj)
-    except (OSError, TypeError):
-        lineno = None
-
-    linespec = f"#L{lineno:d}-L{lineno + len(source) - 1:d}" if lineno else ""
-
-    startdir = Path(flexeval.__file__).parent.parent
-    try:
-        fn = os.path.relpath(fn, start=startdir).replace(os.path.sep, "/")
-    except ValueError:
-        return None
-
-    if not fn.startswith(("matplotlib/", "mpl_toolkits/")):
-        return None
-
-    version = parse_version(flexeval.__version__)
-    tag = "main" if version.is_devrelease else f"v{version.public}"
-    return f"https://github.com/matplotlib/matplotlib/blob/{tag}/lib/{fn}{linespec}"
-
-
 # myst-parser
 # https://myst-parser.readthedocs.io/en/latest/configuration.html
 myst_gfm_only = False
@@ -170,12 +95,6 @@ def linkcode_resolve(domain, info):
 nb_merge_streams = True
 
 autosummary_generate = True
-# Don't let numpydoc inject its own per-class "Methods"/"Attributes" summary
-# tables. They duplicate the member documentation autodoc already renders below,
-# and for our peewee models they list every inherited peewee.Model method
-# (save, select, bulk_create, ...) as noise. Disabling this also avoids the
-# "stub file not found" warnings those tables' :toctree: would otherwise emit.
-numpydoc_show_class_members = False
 autodoc_typehints = "signature"
 # Some models hold fields that have no JSON-schema representation (e.g.
 # FunctionsCollection.functions is a list[Callable]). Coerce rather than warn so

pyproject.toml

@@ -79,13 +79,10 @@ docs = [
     "linkify-it-py>=2.0.3",
     "myst-nb>=1.2.0",
     "myst-parser>=4.0.1",
-    "numpydoc>=1.9.0",
     "pydata-sphinx-theme>=0.16.1",
     "sphinx>=8.1.3",
     "sphinx-autobuild>=2024.10.3",
     "sphinx-copybutton>=0.5.2",
-    "sphinx-design>=0.6.1",
-    "sphinx-tags>=0.4",
     "sphinxcontrib-programoutput>=0.18",
 ]