prepare for 1.16.0rc1

* add support for python 3.12
* mask reimport warning noise in tests
* fix/skip/xfail tests broken by Python 3.12
* xfail broken Python 3.8 tests broken by newer macOS releases
* drop support for EOL Pythons (< 3.8)
* declare python_requires metadata for 3.8+ (allows unsupported Pythons
  to continue using previous sdist/wheel releases indefinitely)
* fix musllinux 1.1 tests
* add boilerplate pyproject.toml for basic PEP517 build support
* add explicit build-time dependency on setuptools
* shim runtime setuptools/distutils imports to provide more
  prescriptive error messages when missing or broken
* update CI configurations
* add .gitignore to prepare for move to GitHub

Author: nitzmahone

.github/workflows/ci.yaml

@@ -5,7 +5,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: clone repo
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: build sdist
       run: |
@@ -14,48 +14,55 @@ jobs:
         python -m build --sdist
 
     - name: upload sdist artifact
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         path: dist
         if-no-files-found: error
 
   linux:
     runs-on: ubuntu-20.04
     strategy:
+      fail-fast: false
       matrix:
         include:
-        - spec: cp27-manylinux_x86_64
-          cibw_version: cibuildwheel<2.0  # py2.7 is not supported on CIBW 2.0+
-          manylinux_img: manylinux1  # build really old Pythons on manylinux1
-        - spec: cp36-manylinux_x86_64
-          manylinux_img: manylinux1  # build really old Pythons on manylinux1
-        - spec: cp37-manylinux_x86_64
         - spec: cp38-manylinux_x86_64
         - spec: cp39-manylinux_x86_64
         - spec: cp310-manylinux_x86_64
         - spec: cp311-manylinux_x86_64
-        - spec: cp27-manylinux_i686
-          cibw_version: cibuildwheel<2.0  # py2.7 is not supported on CIBW 2.0+
-          manylinux_img: manylinux1  # build really old Pythons on manylinux1
-        - spec: cp36-manylinux_i686
-          manylinux_img: manylinux1  # build really old Pythons on manylinux1
-        - spec: cp37-manylinux_i686
+        - spec: cp312-manylinux_x86_64
+
         - spec: cp38-manylinux_i686
         - spec: cp39-manylinux_i686
         - spec: cp310-manylinux_i686
         - spec: cp311-manylinux_i686
+        - spec: cp312-manylinux_i686
+
         - spec: cp39-musllinux_x86_64
         - spec: cp310-musllinux_x86_64
         - spec: cp311-musllinux_x86_64
+        - spec: cp312-musllinux_x86_64
+
         - spec: cp39-musllinux_i686
         - spec: cp310-musllinux_i686
         - spec: cp311-musllinux_i686
-        - spec: cp36-manylinux_ppc64le
+        #- spec: cp312-musllinux_i686  # busted as of 9/22/23
+
+        - spec: cp38-manylinux_aarch64
+          foreign_arch: true
+          test_args: '{project}/c'
+        - spec: cp39-manylinux_aarch64
+          foreign_arch: true
+          test_args: '{project}/c'
+        - spec: cp310-manylinux_aarch64
+          foreign_arch: true
+          test_args: '{project}/c'
+        - spec: cp311-manylinux_aarch64
           foreign_arch: true
           test_args: '{project}/c'
-        - spec: cp37-manylinux_ppc64le
+        - spec: cp312-manylinux_aarch64
           foreign_arch: true
           test_args: '{project}/c'
+
         - spec: cp38-manylinux_ppc64le
           foreign_arch: true
           test_args: '{project}/c'
@@ -68,9 +75,29 @@ jobs:
         - spec: cp311-manylinux_ppc64le
           foreign_arch: true
           test_args: '{project}/c'
+        - spec: cp312-manylinux_ppc64le
+          foreign_arch: true
+          test_args: '{project}/c'
+
+        - spec: cp38-manylinux_s390x
+          foreign_arch: true
+          test_args: '{project}/c'
+        - spec: cp39-manylinux_s390x
+          foreign_arch: true
+          test_args: '{project}/c'
+        - spec: cp310-manylinux_s390x
+          foreign_arch: true
+          test_args: '{project}/c'
+        - spec: cp311-manylinux_s390x
+          foreign_arch: true
+          test_args: '{project}/c'
+        - spec: cp312-manylinux_s390x
+          foreign_arch: true
+          test_args: '{project}/c'
+
     steps:
     - name: clone repo
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: configure docker foreign arch support
       uses: docker/setup-qemu-action@v1
@@ -94,7 +121,7 @@ jobs:
         CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_img || '' }}
         CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_img || '' }}
         CIBW_PRERELEASE_PYTHONS: 'True'
-        CIBW_TEST_REQUIRES: pytest
+        CIBW_TEST_REQUIRES: pytest setuptools  # 3.12+ no longer includes distutils, just always ensure setuptools is present
         CIBW_TEST_COMMAND: PYTHONUNBUFFERED=1 python -m pytest ${{ matrix.test_args || '{project}' }}  # default to test all
       run: |
         python -m pip install --upgrade "${{ matrix.cibw_version || 'cibuildwheel' }}"
@@ -104,7 +131,7 @@ jobs:
 
 
     - name: upload artifacts
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         path: dist
         if-no-files-found: error
@@ -114,21 +141,19 @@ jobs:
     defaults:
       run:
         shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }}
-    runs-on: ${{ matrix.runs_on || 'macos-10.15' }}
+    runs-on: ${{ matrix.runs_on || 'macos-11' }}
     strategy:
+      fail-fast: false
       matrix:
         include:
         # build for x86_64 under the default hosted macOS 10.x x86_64 runner
-        - spec: cp27-macosx_x86_64
-          cibw_version: cibuildwheel<2.0  # py2.7 is not supported on CIBW 2.0+
-        - spec: cp36-macosx_x86_64
-        - spec: cp37-macosx_x86_64
         - spec: cp38-macosx_x86_64
         - spec: cp39-macosx_x86_64
         - spec: cp310-macosx_x86_64
         - spec: cp311-macosx_x86_64
-        # build for arm64 under a hacked macOS 12 self-hosted x86_64-on-arm64 runner until arm64 is fully supported
-        # FIXME: ? cp38-macosx_arm64 requires special handling and fails some test_zdist tests under cibw 2.1.2, skip it (so Apple's XCode python3 won't have a wheel)
+        - spec: cp312-macosx_x86_64
+#        # build for arm64 under a hacked macOS 12 self-hosted x86_64-on-arm64 runner until arm64 is fully supported
+#        # FIXME: ? cp38-macosx_arm64 requires special handling and fails some test_zdist tests under cibw 2.1.2, skip it (so Apple's XCode python3 won't have a wheel)
         - spec: cp39-macosx_arm64
           deployment_target: '11.0'
           runs_on: [self-hosted, macOS]
@@ -147,9 +172,16 @@ jobs:
           run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
           sdkroot: macosx11.3
 
+        - spec: cp312-macosx_arm64
+          deployment_target: '11.0'
+          runs_on: [self-hosted, macOS]
+          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
+          sdkroot: macosx11.3
+
 
     steps:
     - name: clone repo
+      # need to use v2 until we can upgrade the runners on our private Apple Silicon build infra to one that supports node20
       uses: actions/checkout@v2
 
     - name: build wheel prereqs
@@ -161,7 +193,7 @@ jobs:
       env:
         CIBW_BUILD: ${{ matrix.spec }}
         CIBW_PRERELEASE_PYTHONS: 'True'
-        CIBW_TEST_REQUIRES: pytest
+        CIBW_TEST_REQUIRES: pytest setuptools
         CIBW_TEST_COMMAND: pip install pip --upgrade; cd {project}; PYTHONUNBUFFERED=1 pytest
       run: |
         if [[ -n "${{ matrix.deployment_target || '' }}" ]]
@@ -177,6 +209,7 @@ jobs:
         /usr/bin/python3 -m cibuildwheel --output-dir dist
 
     - name: upload artifacts
+      # need to use v2 until we can upgrade the runners on our private Apple Silicon build infra to one that supports node20
       uses: actions/upload-artifact@v2
       with:
         path: dist
@@ -185,56 +218,39 @@ jobs:
   windows:
     runs-on: windows-2019
     strategy:
+      fail-fast: false
       matrix:
         include:
-        - spec: cp27-win_amd64
-          cibw_version: cibuildwheel==1.10  # last release with proper py2.7 Windows support
-        - spec: cp36-win_amd64
-        - spec: cp37-win_amd64
         - spec: cp38-win_amd64
         - spec: cp39-win_amd64
         - spec: cp310-win_amd64
         - spec: cp311-win_amd64
-        - spec: cp27-win32
-          cibw_version: cibuildwheel==1.10  # last release with proper py2.7 Windows support
-        - spec: cp36-win32
-        - spec: cp37-win32
+        - spec: cp312-win_amd64
         - spec: cp38-win32
         - spec: cp39-win32
         - spec: cp310-win32
         - spec: cp311-win32
+        - spec: cp312-win32
+
     steps:
     - name: clone repo
-      uses: actions/checkout@v2
-
-    # HACK: MS killed this SDK support package and removed downloads, as did chocolatey, install from a private archive
-    - name: install Windows Python 2.7 SDK
-      run: |
-        $msiPath = Join-Path ([IO.Path]::GetTempPath()) 'VCForPython27.msi'
-        [Net.WebClient]::new().DownloadFile('https://raw.githubusercontent.com/nitzmahone/VCPython27/master/VCForPython27.msi', $msiPath)
-        Start-Process -FilePath msiexec.exe -Wait -ArgumentList @(
-            '/i',
-            $msiPath,
-            '/qn',
-            '/norestart'
-        )
-      shell: powershell
-      if: ${{ contains(matrix.spec, 'cp27') }}
-
+      uses: actions/checkout@v4
     - name: build/test wheels
       env:
         CIBW_BUILD: ${{ matrix.spec }}
         CIBW_PRERELEASE_PYTHONS: 'True'
+        CIBW_TEST_REQUIRES: pytest setuptools
+        CIBW_TEST_COMMAND: 'python -m pytest {project}/c'
+        # FIXME: /testing takes ~45min on Windows and has some failures...
+        # CIBW_TEST_COMMAND='python -m pytest {project}/c {project}/testing'
       run: |
         python -m pip install --upgrade pip
         pip install "${{ matrix.cibw_version || 'cibuildwheel'}}"
-        # FIXME: /testing takes ~45min on Windows and has some failures...
-        # CIBW_TEST_REQUIRES=pytest CIBW_TEST_COMMAND='python -m pytest {project}/c {project}/testing' cibuildwheel --output-dir dist .
-        CIBW_TEST_REQUIRES=pytest CIBW_TEST_COMMAND='python -m pytest {project}/c' cibuildwheel --output-dir dist .
+        python -m cibuildwheel --output-dir dist .
       shell: bash
 
     - name: upload artifacts
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         path: dist
         if-no-files-found: error

.gitignore

@@ -0,0 +1,6 @@
+build/
+/dist/
+*.py[cod]
+__pycache__/
+*.egg-info/
+*.so

c/test_c.py

@@ -75,6 +75,8 @@ def find_and_load_library(name, flags=RTLD_NOW):
         path = None
     else:
         path = ctypes.util.find_library(name)
+        if path is None and sys.platform == 'darwin' and sys.version_info[:2] == (3, 8):
+            pytest.xfail("find_library usually broken on MacOS Python 3.8")
         if path is None and name == 'c':
             assert sys.platform == 'win32'
             assert (sys.version_info >= (3,) or

cffi/_shimmed_dist_utils/__init__.py

@@ -0,0 +1,41 @@
+"""
+Temporary shim module to indirect the bits of distutils we need from setuptools/distutils while providing useful
+error messages beyond `No module named 'distutils' on Python >= 3.12, or when setuptools' vendored distutils is broken.
+
+This is a compromise to avoid a hard-dep on setuptools for Python >= 3.12, since many users don't need runtime compilation support from CFFI.
+"""
+import sys
+
+try:
+    # import setuptools first; this is the most robust way to ensure its embedded distutils is available
+    # (the .pth shim should usually work, but this is even more robust)
+    import setuptools
+except Exception as ex:
+    if sys.version_info >= (3, 12):
+        # Python 3.12 has no built-in distutils to fall back on, so any import problem is fatal
+        raise Exception("This CFFI feature requires setuptools on Python >= 3.12. The setuptools module is missing or non-functional.") from ex
+
+    # silently ignore on older Pythons (support fallback to stdlib distutils where available)
+else:
+    del setuptools
+
+try:
+    # bring in just the bits of distutils we need, whether they really came from setuptools or stdlib-embedded distutils
+    from distutils import log, sysconfig
+    from distutils.ccompiler import CCompiler
+    from distutils.command.build_ext import build_ext
+    from distutils.core import Distribution, Extension
+    from distutils.dir_util import mkpath
+    from distutils.errors import DistutilsSetupError, CompileError, LinkError
+    from distutils.log import set_threshold, set_verbosity
+
+    if sys.platform == 'win32':
+        from distutils.msvc9compiler import MSVCCompiler
+except Exception as ex:
+    if sys.version_info >= (3, 12):
+        raise Exception("This CFFI feature requires setuptools on Python >= 3.12. Please install the setuptools package.") from ex
+
+    # anything older, just let the underlying distutils import error fly
+    raise Exception("This CFFI feature requires distutils. Please install the distutils or setuptools package.") from ex
+
+del sys

cffi/api.py

@@ -622,7 +622,7 @@ def ensure(key, value):
                 try:
                     import sysconfig
                 except ImportError:    # 2.6
-                    from distutils import sysconfig
+                    from cffi._shimmed_dist_utils import sysconfig
                 template = "python%d.%d"
                 if sysconfig.get_config_var('DEBUG_EXT'):
                     template += sysconfig.get_config_var('DEBUG_EXT')
@@ -658,7 +658,7 @@ def set_source_pkgconfig(self, module_name, pkgconfig_libs, source,
         self.set_source(module_name, source, source_extension, **kwds)
 
     def distutils_extension(self, tmpdir='build', verbose=True):
-        from distutils.dir_util import mkpath
+        from cffi._shimmed_dist_utils import mkpath
         from .recompiler import recompile
         #
         if not hasattr(self, '_assigned_source'):

cffi/ffiplatform.py

@@ -6,8 +6,7 @@
                       'extra_objects', 'depends']
 
 def get_extension(srcfilename, modname, sources=(), **kwds):
-    _hack_at_distutils()
-    from distutils.core import Extension
+    from cffi._shimmed_dist_utils import Extension
     allsources = [srcfilename]
     for src in sources:
         allsources.append(os.path.normpath(src))
@@ -16,7 +15,6 @@ def get_extension(srcfilename, modname, sources=(), **kwds):
 def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
-    _hack_at_distutils()
     saved_environ = os.environ.copy()
     try:
         outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
@@ -31,9 +29,8 @@ def compile(tmpdir, ext, compiler_verbose=0, debug=None):
 
 def _build(tmpdir, ext, compiler_verbose=0, debug=None):
     # XXX compact but horrible :-(
-    from distutils.core import Distribution
-    import distutils.errors, distutils.log
-    #
+    from cffi._shimmed_dist_utils import Distribution, CompileError, LinkError, set_threshold, set_verbosity
+
     dist = Distribution({'ext_modules': [ext]})
     dist.parse_config_files()
     options = dist.get_option_dict('build_ext')
@@ -45,16 +42,15 @@ def _build(tmpdir, ext, compiler_verbose=0, debug=None):
     options['build_temp'] = ('ffiplatform', tmpdir)
     #
     try:
-        old_level = distutils.log.set_threshold(0) or 0
+        old_level = set_threshold(0) or 0
         try:
-            distutils.log.set_verbosity(compiler_verbose)
+            set_verbosity(compiler_verbose)
             dist.run_command('build_ext')
             cmd_obj = dist.get_command_obj('build_ext')
             [soname] = cmd_obj.get_outputs()
         finally:
-            distutils.log.set_threshold(old_level)
-    except (distutils.errors.CompileError,
-            distutils.errors.LinkError) as e:
+            set_threshold(old_level)
+    except (CompileError, LinkError) as e:
         raise VerificationError('%s: %s' % (e.__class__.__name__, e))
     #
     return soname
@@ -115,13 +111,3 @@ def flatten(x):
     f = cStringIO.StringIO()
     _flatten(x, f)
     return f.getvalue()
-
-def _hack_at_distutils():
-    # Windows-only workaround for some configurations: see
-    # https://bugs.python.org/issue23246 (Python 2.7 with 
-    # a specific MS compiler suite download)
-    if sys.platform == "win32":
-        try:
-            import setuptools    # for side-effects, patches distutils
-        except ImportError:
-            pass