Host the document using RTD. (#4)

- Build a CPU-only pyhwloc in the sphinx conf.
- Deduplicate the commit hash.

Author: trivialfis

.github/workflows/main.yml

@@ -193,9 +193,10 @@ jobs:
     - name: Install hwloc
       shell: pwsh
       run: |
+        $Env:HWLOC_VERSION = $(cat ./dev/hwloc_version)
         git clone https://github.com/open-mpi/hwloc.git
         cd hwloc
-        git checkout 82d8d2c1ad17e0b48f046f997271e45946bd342e
+        git checkout $Env:HWLOC_VERSION
 
         cd contrib/windows-cmake
         mkdir build

.readthedocs.yaml

@@ -8,15 +8,13 @@ version: 2
 build:
   os: ubuntu-24.04
   tools:
-    python: "3.12"
+    python: "miniconda-latest"
+  apt_packages:
+    - doxygen
 
 # Build documentation in the "docs/" directory with Sphinx
 sphinx:
    configuration: docs/source/conf.py
 
-# Optionally, but recommended,
-# declare the Python requirements required to build your documentation
-# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-# python:
-#    install:
-#    - requirements: docs/requirements.txt
+conda:
+  environment: dev/pyhwloc_dev.yml

CMakeLists.txt

@@ -8,6 +8,7 @@ project(pyhwloc LANGUAGES C VERSION 0.0.0)
 list(APPEND CMAKE_MODULE_PATH "${pyhwloc_SOURCE_DIR}/cmake/")
 
 option(PYHWLOC_FETCH_HWLOC "Fetch hwloc from GitHub instead of using system version" OFF)
+option(PYHWLOC_WITH_CUDA "Add CUDA-related components" ON)
 
 set(PYHWLOC_BUILD_HWLOC_CMAKE OFF)
 if(WIN32)
@@ -23,10 +24,12 @@ if(PYHWLOC_FETCH_HWLOC)
   set(PYHWLOC_CONFIG_HWLOC OFF CACHE BOOL "")
 
   include(FetchContent)
+  file(READ ${pyhwloc_SOURCE_DIR}/dev/hwloc_version HWLOC_VERSION)
+  message(STATUS "HWLOC_VERSION: ${HWLOC_VERSION}")
   FetchContent_Declare(
     hwloc
     GIT_REPOSITORY https://github.com/open-mpi/hwloc.git
-    GIT_TAG        82d8d2c1ad17e0b48f046f997271e45946bd342e
+    GIT_TAG        ${HWLOC_VERSION}
   )
 
   # Configure hwloc build options
@@ -41,7 +44,7 @@ if(PYHWLOC_FETCH_HWLOC)
         -G ${CMAKE_GENERATOR}
         -DCMAKE_INSTALL_PREFIX=${PYHWLOC_OUTPUT_DIR}
         -DBUILD_SHARED_LIBS=ON
-        -DHWLOC_WITH_CUDA=ON
+        -DHWLOC_WITH_CUDA=${PYHWLOC_WITH_CUDA}
         -DHWLOC_ENABLE_PLUGINS=ON
         WORKING_DIRECTORY ${hwloc_SOURCE_DIR}
       )
@@ -93,26 +96,29 @@ else()
 endif()
 
 message(STATUS "HWLOC_LIBRARY: ${HWLOC_LIBRARY}")
-find_package(CUDAToolkit REQUIRED COMPONENTS nvml cuda_driver cudart_static)
 
 target_link_libraries(pyhwloc PRIVATE ${HWLOC_LIBRARY})
 target_include_directories(pyhwloc PRIVATE ${HWLOC_INCLUDE_DIR})
 list(APPEND PYHWLOC_LIBS pyhwloc)
 
-add_library(pyhwloc_cuda SHARED src/ext/cudr.c)
-target_link_libraries(pyhwloc_cuda PRIVATE CUDA::cuda_driver ${HWLOC_LIBRARY})
-target_include_directories(pyhwloc_cuda PRIVATE ${HWLOC_INCLUDE_DIR})
-list(APPEND PYHWLOC_LIBS pyhwloc_cuda)
+if(PYHWLOC_WITH_CUDA)
+  find_package(CUDAToolkit REQUIRED COMPONENTS nvml cuda_driver cudart_static)
 
-add_library(pyhwloc_cudart SHARED src/ext/cudart.c)
-target_link_libraries(pyhwloc_cudart PRIVATE CUDA::cudart ${HWLOC_LIBRARY})
-target_include_directories(pyhwloc_cudart PRIVATE ${HWLOC_INCLUDE_DIR})
-list(APPEND PYHWLOC_LIBS pyhwloc_cudart)
+  add_library(pyhwloc_cuda SHARED src/ext/cudr.c)
+  target_link_libraries(pyhwloc_cuda PRIVATE CUDA::cuda_driver ${HWLOC_LIBRARY})
+  target_include_directories(pyhwloc_cuda PRIVATE ${HWLOC_INCLUDE_DIR})
+  list(APPEND PYHWLOC_LIBS pyhwloc_cuda)
 
-add_library(pyhwloc_nvml SHARED src/ext/nvml.c)
-target_link_libraries(pyhwloc_nvml PRIVATE CUDA::nvml ${HWLOC_LIBRARY})
-target_include_directories(pyhwloc_nvml PRIVATE ${HWLOC_INCLUDE_DIR})
-list(APPEND PYHWLOC_LIBS pyhwloc_nvml)
+  add_library(pyhwloc_cudart SHARED src/ext/cudart.c)
+  target_link_libraries(pyhwloc_cudart PRIVATE CUDA::cudart ${HWLOC_LIBRARY})
+  target_include_directories(pyhwloc_cudart PRIVATE ${HWLOC_INCLUDE_DIR})
+  list(APPEND PYHWLOC_LIBS pyhwloc_cudart)
+
+  add_library(pyhwloc_nvml SHARED src/ext/nvml.c)
+  target_link_libraries(pyhwloc_nvml PRIVATE CUDA::nvml ${HWLOC_LIBRARY})
+  target_include_directories(pyhwloc_nvml PRIVATE ${HWLOC_INCLUDE_DIR})
+  list(APPEND PYHWLOC_LIBS pyhwloc_nvml)
+endif()
 
 include(GenerateExportHeader)
 

README.rst

@@ -1,4 +1,5 @@
 Python Interface for the Portable Hardware Locality (hwloc) Library
 ===================================================================
 
-- `Official site <https://www.open-mpi.org/projects/hwloc/>`__ of the hwloc library.
+- `Document <https://pyhwloc.readthedocs.io/>`__.
+- `Official site <https://www.open-mpi.org/projects/hwloc/>`__ of the hwloc library.

dev/Dockerfile.cpu

@@ -18,10 +18,12 @@ RUN \
     git clone https://github.com/doxygen/doxygen.git && \
     cd doxygen && git checkout d49be63057d71f487312f43339516bc92cc685fa && cd -
 
+COPY hwloc_version ws/hwloc_version
+
 RUN \
     cd /ws/ && \
     git clone https://github.com/open-mpi/hwloc.git && \
-    cd hwloc && git checkout 82d8d2c1ad17e0b48f046f997271e45946bd342e && cd -
+    cd hwloc && git checkout $(cat /ws/hwloc_version) && cd -
 
 SHELL ["mamba", "run", "--no-capture-output", "-n", "base", "/bin/bash", "-c"]
 

dev/hwloc_version

@@ -0,0 +1 @@
+82d8d2c1ad17e0b48f046f997271e45946bd342e

docs/source/build.rst

@@ -99,8 +99,11 @@ A complete list of options available for the ``--config-settings=``:
 - ``build-dir=/path/to/build/dir`` for specifying a build dir.
 - ``hwloc-src-dir=/path/to/hwloc-src`` for using a local checkout of hwloc. This assumes
   the src directory is the git repo, which is not the same as the release tarball.
-- ``hwloc-root-dir=/path/to/hwloc`` to specify the path of an existing hwloc installation.
+- ``hwloc-root-dir=/path/to/hwloc`` to specify the path of an existing binary hwloc
+  installation.
 - ``fetch-hwloc=True`` to build the fat wheel.
+- ``with-cuda=False`` to build pyhwloc without CUDA plugins. This is used for building
+  document in a minimum environment and is not tested for other purposes.
 
 The binary wheel uses plugins by default. Due to the plugins support, all symbols from
 hwloc are loaded into the linker's public name space using

docs/source/conf.py

@@ -9,6 +9,11 @@
 from __future__ import annotations
 
 import os
+import subprocess
+import sys
+import tempfile
+from contextlib import contextmanager
+from typing import Iterator
 
 project = "pyhwloc"
 copyright = "2025, Jiaming Yuan"
@@ -36,21 +41,6 @@
 
 intersphinx_mapping = {"python": ("https://docs.python.org/3.10", None)}
 
-# -- Breathe
-breathe_default_project = "pyhwloc"
-breathe_domain_by_extension = {"h": "c"}
-
-CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))  # source
-PROJECT_ROOT = os.path.normpath(os.path.join(CURR_PATH, os.path.pardir, os.path.pardir))
-
-hwloc_xml_path = os.environ.get("PYHWLOC_XML_PATH", None)
-if hwloc_xml_path is None:
-    hwloc_xml_path = os.path.join(
-        PROJECT_ROOT, os.path.pardir, "hwloc/doc/doxygen-doc/xml"
-    )
-breathe_projects = {"pyhwloc": hwloc_xml_path}
-print("beathe projects", breathe_projects)
-
 # -- Build environment
 
 os.environ["PYHWLOC_SPHINX"] = "1"
@@ -62,3 +52,103 @@
     # path to where to save gallery generated output
     "gallery_dirs": ["examples"],
 }
+
+
+def is_readthedocs_build() -> bool:
+    if os.environ.get("READTHEDOCS", None) == "True":
+        return True
+    return False
+
+
+def normpath(path: str) -> str:
+    return os.path.normpath(os.path.abspath(path))
+
+
+@contextmanager
+def _chdir(dirname: str) -> Iterator[None]:
+    pwd = normpath(os.path.curdir)
+    try:
+        os.chdir(dirname)
+        yield
+    finally:
+        os.chdir(pwd)
+
+
+def build_pyhwloc_xml() -> str:
+    """Build and install pyhwloc, returns the path to the doxygen xml files."""
+    if sys.platform == "win32":
+        raise NotImplementedError("Read the docs environment should be Linux.")
+
+    hwloc_version_path = os.path.join(
+        PROJECT_ROOT,
+        "dev",
+        "hwloc_version",
+    )
+    with open(hwloc_version_path, "r") as fd:
+        hwloc_version = fd.read().strip()
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        pwd = normpath(os.path.curdir)
+        script = f"""#!/usr/bin/env bash
+# Clone
+git clone https://github.com/open-mpi/hwloc.git
+cd hwloc
+git checkout {hwloc_version}
+
+# Config
+./autogen.sh
+./configure --disable-nvml --enable-doxygen
+
+# Build doc
+cd doc
+HWLOC_DOXYGEN_GENERATE_XML=YES doxygen ./doxygen.cfg
+# Result is in `hwloc/doc/doxygen-doc/xml`
+
+# Copy and cleanup
+cp -r doxygen-doc/xml {pwd}/
+cd ..
+git clean -xdf
+"""
+        script_path = os.path.join(tmpdir, "build_xml.sh")
+        with open(script_path, "w") as fd:
+            fd.write(script)
+
+        with _chdir(tmpdir):
+            subprocess.check_call(["bash", script_path])
+            xml_path = os.path.join(pwd, "xml")
+
+        # Install pyhwloc while we have the hwloc source
+        hwloc_src_dir = os.path.join(tmpdir, "hwloc")
+        subprocess.check_call(
+            [
+                "pip",
+                "install",
+                PROJECT_ROOT,
+                "--config-settings=fetch-hwloc=True",
+                f"--config-settings=hwloc-src-dir={hwloc_src_dir}",
+                "--config-settings=with-cuda=False",
+                "--no-deps",
+                "--no-build-isolation",
+            ]
+        )
+
+    return xml_path
+
+
+# -- Breathe
+breathe_default_project = "pyhwloc"
+breathe_domain_by_extension = {"h": "c"}
+
+CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))  # source
+PROJECT_ROOT = os.path.normpath(os.path.join(CURR_PATH, os.path.pardir, os.path.pardir))
+
+if is_readthedocs_build():
+    hwloc_xml_path: str | None = build_pyhwloc_xml()
+else:
+    hwloc_xml_path = os.environ.get("PYHWLOC_XML_PATH", None)
+    if hwloc_xml_path is None:
+        hwloc_xml_path = os.path.join(
+            PROJECT_ROOT, os.path.pardir, "hwloc/doc/doxygen-doc/xml"
+        )
+breathe_projects = {"pyhwloc": hwloc_xml_path}
+print("breathe projects", breathe_projects)

docs/source/dev.rst

@@ -8,6 +8,11 @@ Symbol Conflicts
 The hwloc is loaded into the public linker name space to support hwloc plugins. This might
 have unintended consequences.
 
+Update hwloc
+============
+
+Update the commit hash in ``dev/hwloc_version``.
+
 Design Decisions
 ================
 

hatch/backend.py

@@ -10,7 +10,7 @@
 
 import hatchling.build
 
-from .hook import BUILD_KEY, FETCH_KEY, ROOT_KEY, SRC_KEY
+from .hook import BUILD_KEY, CUDA_KEY, FETCH_KEY, ROOT_KEY, SRC_KEY
 
 
 @contextmanager
@@ -26,6 +26,10 @@ def build_config(config_settings: dict[str, Any] | None) -> Iterator[None]:
             os.environ[SRC_KEY] = config_settings["hwloc-src-dir"]
         if "hwloc-root-dir" in config_settings:
             os.environ[ROOT_KEY] = config_settings["hwloc-root-dir"]
+        if "with-cuda" in config_settings:
+            v = config_settings["with-cuda"]
+            assert v in ("True", "False")
+            os.environ[CUDA_KEY] = v
     try:
         yield
     finally:
@@ -37,6 +41,8 @@ def build_config(config_settings: dict[str, Any] | None) -> Iterator[None]:
             del os.environ[SRC_KEY]
         if ROOT_KEY in os.environ:
             del os.environ[ROOT_KEY]
+        if CUDA_KEY in os.environ:
+            del os.environ[CUDA_KEY]
 
 
 def build_wheel(