MAINT - Ensure Playwright tests use test sites and are run in CI

pyproject.toml

@@ -81,7 +81,6 @@ doc = [
   "ipywidgets",
   "graphviz",
 ]
-test = ["pytest", "pytest-cov", "pytest-regressions", "sphinx[test]"]
 dev = [
   "pyyaml",
   "pre-commit",
@@ -90,7 +89,13 @@ dev = [
   "pandoc",
   "sphinx-theme-builder[cli]",
 ]
-a11y = ["pytest-playwright"]
+test = [
+  "pytest",
+  "pytest-cov",
+  "pytest-regressions",
+  "sphinx[test]",
+  "pytest-playwright"
+]
 i18n = ["Babel", "jinja2"]
 
 [project.entry-points]
@@ -109,7 +114,7 @@ indent-width = 4
 ignore = [
   "D107", # Missing docstring in `__init__` | set the docstring in the class
   "D205", # 1 blank line required between summary line and description,
-  "D212",  # docstring summary must be on first physical line
+  "D212", # docstring summary must be on first physical line
   "W291", # let pre-commit handle trailing whitespace
 
 ]

tests/sites/version_switcher/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

tests/sites/version_switcher/_static/switcher.json

@@ -0,0 +1,11 @@
+[
+  {
+    "version": "dev",
+    "url": "https://pydata-sphinx-theme.readthedocs.io/en/latest/"
+  },
+  {
+    "name": "0.16.1 (stable)",
+    "version": "v0.16.1",
+    "url": "https://pydata-sphinx-theme.readthedocs.io/en/stable/"
+  }
+]

tests/sites/version_switcher/conf.py

@@ -0,0 +1,22 @@
+"""Test conf file - basic site with version switcher."""
+
+# -- Project information -----------------------------------------------------
+project = "PyData Tests"
+copyright = "2020, Pydata community"
+author = "Pydata community"
+
+root_doc = "index"
+
+# -- General configuration ---------------------------------------------------
+html_theme = "pydata_sphinx_theme"
+
+html_static_path = ["_static"]
+
+# Add version switcher
+html_theme_options = {
+    "switcher": {
+        "json_url": "_static/switcher.json",
+        "version_match": "dev",
+    },
+    "navbar_start": ["navbar-logo", "version-switcher"],
+}

tests/sites/version_switcher/index.rst

@@ -0,0 +1,15 @@
+Simple site with version switcher
+=================================
+
+.. toctree::
+   :caption: Caption 1
+   :numbered:
+
+   page1
+   page2
+   section1/index
+
+Other content
+-------------
+
+Nothing to see here

tests/sites/version_switcher/page1.rst

@@ -0,0 +1,33 @@
+Page 1
+======
+
+**normal link**
+
+- https://pydata-sphinx-theme.readthedocs.io/en/latest/
+
+**GitHub**
+
+.. container:: github-container
+
+    https://github.com
+    https://github.com/pydata
+    https://github.com/pydata/pydata-sphinx-theme
+    https://github.com/pydata/pydata-sphinx-theme/pull/1012
+    https://github.com/orgs/pydata/projects/2
+
+**GitLab**
+
+.. container:: gitlab-container
+
+    https://gitlab.com
+    https://gitlab.com/gitlab-org
+    https://gitlab.com/gitlab-org/gitlab
+    https://gitlab.com/gitlab-org/gitlab/-/issues/375583
+    https://gitlab.com/gitlab-org/gitlab/issues/375583
+    https://gitlab.com/gitlab-org/gitlab/-/issues/
+    https://gitlab.com/gitlab-org/gitlab/issues/
+    https://gitlab.com/gitlab-org/gitlab/-/issues
+    https://gitlab.com/gitlab-org/gitlab/issues
+    https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84669
+    https://gitlab.com/gitlab-org/gitlab/-/pipelines/511894707
+    https://gitlab.com/gitlab-com/gl-infra/production/-/issues/6788

tests/sites/version_switcher/page2.rst

@@ -0,0 +1,9 @@
+:html_theme.sidebar_secondary.remove: true
+
+Page :math:`\beta`
+==================
+
+.. code-block:: python
+
+    import pydata_sphinx_theme as pst
+    raise RuntimeError('Test of pygments highlighting')

tests/sites/version_switcher/section1/index.rst

@@ -0,0 +1,6 @@
+Section 1 index
+===============
+.. toctree::
+
+   page1
+   https://google.com

tests/sites/version_switcher/section1/page1.rst

@@ -0,0 +1,2 @@
+Section 1 page1
+===============

tests/test_a11y.py

@@ -1,4 +1,8 @@
-"""Using Axe-core, scan the Kitchen Sink pages for accessibility violations."""
+"""
+Using Axe-core, scan the Kitchen Sink pages for accessibility violations.
+Note that in contrast with the rest of our tests, the accessibility tests in this file
+are run against a build of our PST documentation, not purposedly-built test sites.
+"""
 
 from urllib.parse import urljoin
 
@@ -282,17 +286,6 @@ def test_notebook_ipywidget_output_tab_stop(page: Page, url_base: str) -> None:
     assert ipywidget.evaluate("el => el.tabIndex") == 0
 
 
-def test_breadcrumb_expansion(page: Page, url_base: str) -> None:
-    """Foo."""
-    # page.goto(urljoin(url_base, "community/practices/merge.html"))
-    # expect(page.get_by_label("Breadcrumb").get_by_role("list")).to_contain_text("Merge and review policy") # noqa: E501
-    page.set_viewport_size({"width": 1440, "height": 720})
-    page.goto(urljoin(url_base, "community/topics/config.html"))
-    expect(page.get_by_label("Breadcrumb").get_by_role("list")).to_contain_text(
-        "Update Sphinx configuration during the build"
-    )
-
-
 @pytest.mark.a11y
 def test_search_as_you_type(page: Page, url_base: str) -> None:
     """Search-as-you-type feature should support keyboard navigation.

tests/test_playwright.py

@@ -1,4 +1,8 @@
-"""Build minimal test sites with sphinx_build_factory and test them with Playwright."""
+"""
+Build minimal test sites with sphinx_build_factory and test them with Playwright.
+When adding new tests to this file, remember to also add the corresponding test site
+to `tests/sites/` or use an existing one.
+"""
 
 from pathlib import Path
 from typing import Callable
@@ -21,6 +25,7 @@
 test_sites_dir = repo_path / "docs" / "_build" / "html" / "playwright_tests"
 
 
+# ------------------------- Helper functions -------------------------
 def _is_overflowing(element):
     """Check if an element is being shortened via CSS due to text-overflow property.
 
@@ -42,7 +47,8 @@ def _build_test_site(site_name: str, sphinx_build_factory: Callable) -> None:
 
 def _check_test_site(site_name: str, site_path: Path, test_func: Callable):
     """Make the built test site available to Playwright, then run `test_func` on it."""
-    test_sites_dir.mkdir(exist_ok=True)
+    # Need to ensure parent directories exist in CI
+    test_sites_dir.mkdir(exist_ok=True, parents=True)
     symlink_path = test_sites_dir / site_name
     try:
         symlink_path.symlink_to(site_path, True)
@@ -55,65 +61,39 @@ def _check_test_site(site_name: str, site_path: Path, test_func: Callable):
         test_sites_dir.rmdir()
 
 
-def test_version_switcher_highlighting(page: Page, url_base: str) -> None:
+# ------------------------- Test functions: style -------------------------
+def test_version_switcher_highlighting(
+    sphinx_build_factory: Callable, page: Page, url_base: str
+) -> None:
     """
     In sidebar and topbar - version switcher should apply highlight color to currently
     selected version.
     """
-    page.goto(url=url_base)
-    # no need to include_hidden here ↓↓↓, we just need to get the active version name
-    button = page.get_by_role("button").filter(has_text="dev")
-    active_version_name = button.get_attribute("data-active-version-name")
-    # here we do include_hidden, so sidebar & topbar menus should each have a
-    # matching entry:
-    entries = page.get_by_role("option", include_hidden=True).filter(
-        has_text=active_version_name
-    )
-    assert entries.count() == 2
-    # make sure they're highlighted
-    for entry in entries.all():
-        light_mode = "rgb(10, 125, 145)"  # pst-color-primary
-        # dark_mode = "rgb(63, 177, 197)"
-        expect(entry).to_have_css("color", light_mode)
-
-
-def test_breadcrumb_expansion(page: Page, url_base: str) -> None:
-    """Test breadcrumb text-overflow."""
-    # wide viewport width → no truncation
-    page.set_viewport_size({"width": 1440, "height": 720})
-    page.goto(urljoin(url_base, "community/topics/config.html"))
-    expect(page.get_by_label("Breadcrumb").get_by_role("list")).to_contain_text(
-        "Update Sphinx configuration during the build"
-    )
-    el = page.get_by_text("Update Sphinx configuration during the build").nth(1)
-    expect(el).to_have_css("overflow-x", "hidden")
-    expect(el).to_have_css("text-overflow", "ellipsis")
-    assert not _is_overflowing(el)
-    # narrow viewport width → truncation
-    page.set_viewport_size({"width": 150, "height": 720})
-    assert _is_overflowing(el)
-
-
-def test_breadcrumbs_everywhere(
-    sphinx_build_factory: Callable, page: Page, url_base: str
-) -> None:
-    """Test breadcrumbs truncate properly when placed in various parts of the layout."""
-    site_name = "breadcrumbs"
+    site_name = "version_switcher"
     site_path = _build_test_site(site_name, sphinx_build_factory=sphinx_build_factory)
 
-    def check_breadcrumb_truncation():
-        page.goto(
-            urljoin(url_base, f"playwright_tests/{site_name}/hansel/gretel/house.html")
+    def check_version_switcher_highlighting():
+        page.goto(urljoin(url_base, f"playwright_tests/{site_name}/index.html"))
+        # no need to include_hidden here ↓↓↓, we just need to get the active
+        # version name
+        button = page.get_by_role("button").filter(has_text="dev")
+        active_version_name = button.get_attribute("data-active-version-name")
+
+        active_version_name = button.get_attribute("data-active-version-name")
+        print(active_version_name)
+        # here we do include_hidden, since we are not adding this in the sidebar
+        # we should only get one entry
+        entries = page.get_by_role("option", include_hidden=True).filter(
+            has_text=active_version_name
         )
-        # sidebar should overflow
-        text = "In the oven with my sister, so hot right now. Soooo. Hotttt."
-        el = page.locator("#main-content").get_by_text(text).last
-        assert _is_overflowing(el)
-        # footer containers never trigger ellipsis overflow because min-width is content
-        el = page.locator(".footer-items__center > .footer-item")
-        assert not _is_overflowing(el)
+        assert entries.count() == 1
+        # make sure they're highlighted
+        for entry in entries.all():
+            light_mode = "rgb(10, 125, 145)"  # pst-color-primary
+            # dark_mode = "rgb(63, 177, 197)"
+            expect(entry).to_have_css("color", light_mode)
 
-    _check_test_site(site_name, site_path, check_breadcrumb_truncation)
+    _check_test_site(site_name, site_path, check_version_switcher_highlighting)
 
 
 def test_colors(sphinx_build_factory: Callable, page: Page, url_base: str) -> None:
@@ -143,3 +123,53 @@ def check_colors():
                 expect(el).to_have_css("color", hover_color)
 
     _check_test_site(site_name, site_path, check_colors)
+
+
+# ------------------------- Test functions: layout & components -----------------------
+
+
+def test_breadcrumb_expansion(
+    sphinx_build_factory: Callable, page: Page, url_base: str
+) -> None:
+    """Test breadcrumb text-overflow."""
+    site_name = "breadcrumbs"
+    site_path = _build_test_site(site_name, sphinx_build_factory=sphinx_build_factory)
+
+    def check_breadcrumb_expansion():
+        # wide viewport width → no truncation
+        page.set_viewport_size({"width": 1440, "height": 720})
+        page.goto(urljoin(url_base, "community/topics/config.html"))
+        expect(page.get_by_label("Breadcrumb").get_by_role("list")).to_contain_text(
+            "Update Sphinx configuration during the build"
+        )
+        el = page.get_by_text("Update Sphinx configuration during the build").nth(1)
+        expect(el).to_have_css("overflow-x", "hidden")
+        expect(el).to_have_css("text-overflow", "ellipsis")
+        assert not _is_overflowing(el)
+        # narrow viewport width → truncation
+        page.set_viewport_size({"width": 150, "height": 720})
+        assert _is_overflowing(el)
+
+        _check_test_site(site_name, site_path, check_breadcrumb_expansion)
+
+
+def test_breadcrumbs_everywhere(
+    sphinx_build_factory: Callable, page: Page, url_base: str
+) -> None:
+    """Test breadcrumbs truncate properly when placed in various parts of the layout."""
+    site_name = "breadcrumbs"
+    site_path = _build_test_site(site_name, sphinx_build_factory=sphinx_build_factory)
+
+    def check_breadcrumb_truncation():
+        page.goto(
+            urljoin(url_base, f"playwright_tests/{site_name}/hansel/gretel/house.html")
+        )
+        # sidebar should overflow
+        text = "In the oven with my sister, so hot right now. Soooo. Hotttt."
+        el = page.locator("#main-content").get_by_text(text).last
+        assert _is_overflowing(el)
+        # footer containers never trigger ellipsis overflow because min-width is content
+        el = page.locator(".footer-items__center > .footer-item")
+        assert not _is_overflowing(el)
+
+    _check_test_site(site_name, site_path, check_breadcrumb_truncation)

tox.ini

@@ -15,7 +15,7 @@ env_list =
 # helping contributors run common tasks without needing to call all the steps
 # For example to run the tests: tox run -m tests
 labels =
-    tests = compile-assets, i18n-compile,  py312-tests
+    tests = compile-assets, i18n-compile, py312-tests
     a11y = compile-assets, i18n-compile, py312-docs, a11y-tests
     i18n = i18n-extract, i18n-compile
     live-server = compile-assets, i18n-compile, docs-live
@@ -62,15 +62,20 @@ description = "Run tests Python and Sphinx versions. If a Sphinx version is spec
 # need to ensure the package is installed in editable mode
 package = editable
 extras =
-    test # install dependencies - defined in pyproject.toml
+    test # install dependencies, includes pytest-playwright - defined in pyproject.toml
+pass_env = GITHUB_ACTIONS # so we can check if this is run on GitHub Actions
 deps =
     coverage[toml]
     py39-sphinx61-tests: sphinx~=6.1.0
     py312-sphinxdev: sphinx[test] @ git+https://github.com/sphinx-doc/sphinx.git@master
 depends =
     compile-assets,
     i18n-compile
+allowlist_externals=
+    playwright
+    bash
 commands =
+    bash -c 'if [[ "{env:GITHUB_ACTIONS:}" == "true" ]]; then playwright install --with-deps; else playwright install; fi'
     py3{9,10,11,12}{,-sphinx61,-sphinxdev,}-tests: coverage run -m pytest -m "not a11y" {posargs}
     py3{9,10,11,12}{,-sphinx61,-sphinxdev,}-tests-no-cov: pytest -m "not a11y" {posargs}
 
@@ -82,8 +87,7 @@ description = run accessibility tests with Playwright and axe-core
 base_python = py312 # keep in sync with tests.yml
 pass_env = GITHUB_ACTIONS # so we can check if this is run on GitHub Actions
 extras =
-    test
-    a11y
+    test # install dependencies, includes pytest-playwright - defined in pyproject.toml
 depends =
     compile-assets,
     i18n-compile