Wheels: Deactivate AVX512 in Blosc2 for non-OSX targets (#1763)

* Deactivate AVX512 in Blosc2 for non-OSX targets

* Upgrade Zlib to 1.3.1

Zlib 1.2 is incompatible with CMake 4

* Upgrade zfp to 1.0.1

* Remove ADIOS1 and Blosc1

* Deactivate AVX512 in Blosc2 for non-OSX targets (now actually)

* Same on Windows

* Apply library upgrades to Windows, too

* Upgrade outdated runners

* Apply patch to fix 32-bit builds

* Try some tricks to use git-am

* Use Python's patch module

* Revert "Use Python's patch module"

This reverts commit 4705753.

* setup.py: 0.16.1.post1

* Use git tag for ADIOS2 patch

Author: franzpoeschel

.github/workflows/build.yml

@@ -10,10 +10,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: ubuntu-20.04
+          - os: ubuntu-22.04
             arch: "x86_64"
 
-          - os: ubuntu-20.04
+          - os: ubuntu-22.04
             arch: "i686"
 
           # builds faster on Travis-CI:
@@ -30,14 +30,14 @@ jobs:
           #  arch: "s390x"
 
           # x86-64 (64bit)
-          - os: windows-2019
+          - os: windows-2022
             arch: "AMD64"
 
           # x86 (32bit)
-          - os: windows-2019
+          - os: windows-2022
             arch: "x86"
             env:
-              CMAKE_GENERATOR: "Visual Studio 16 2019"
+              CMAKE_GENERATOR: "Visual Studio 17 2022"
               CMAKE_GENERATOR_PLATFORM: "Win32"
 
           - os: macos-13
@@ -86,6 +86,14 @@ jobs:
         python -m pip install --upgrade pip setuptools wheel
         python -m pip install cibuildwheel==2.21.2
 
+    # 0.16.1.post1 bump
+    - name: Download Patch 1/1
+      uses: suisei-cn/actions-download-file@818d6b7dc8fe73f2f924b6241f2b1134ca1377d9  # 1.6.0
+      id: setupversion
+      with:
+        url: "https://github.com/openPMD/openPMD-api/commit/f94a1d48eefbc81784a9c25f61c913cf30fe2ece.patch"
+        target: src/.patch/
+
 #    # Patch: Fix versioning
 #    - name: Download Patch 1/2
 #      uses: suisei-cn/actions-download-file@818d6b7dc8fe73f2f924b6241f2b1134ca1377d9  # 1.6.0
@@ -102,13 +110,10 @@ jobs:
 #        url: "https://github.com/openPMD/openPMD-api/pull/1684.patch"
 #        target: src/.patch/
 
-#    - name: Apply Patches
-#      run: |
-#        cd src
-#        git apply --exclude=.github/workflows/windows.yml .patch/1680.patch
-#        git apply --exclude=cmake/dependencies/pybind11.cmake .patch/1684.patch
-#        git fetch
-#        git apply --exclude=setup.py -3 .patch/1684.patch
+    - name: Apply Patches
+      run: |
+        cd src
+        git apply .patch/f94a1d48eefbc81784a9c25f61c913cf30fe2ece.patch
 
     - name: Build wheel
       env:

library_builders.bat

@@ -21,6 +21,22 @@ exit /b 0
     https://github.com/ornladios/ADIOS2/archive/v2.10.2.zip
   powershell Expand-Archive adios2-2.10.2.zip -DestinationPath dep-adios2
 
+  curl -sLo dep-adios2/ADIOS2-2.10.2/patch.diff https://github.com/franzpoeschel/ADIOS2/commit/patches-fix-32-bit-builds.patch
+
+  :: Use git-am for applying the patch,
+  :: for some reason, python -m patch just silently does nothing.
+  :: git-am requires a Git repository to apply a patch, but the release zip
+  :: strips away any Git info, so we just quickly initialize a repository.
+  cd dep-adios2/ADIOS2-2.10.2
+  git init
+  git config user.email "[email protected]"
+  git config user.name "Tooling"
+  git add .
+  git commit --message="Initial commit so we can use git-am"
+  git am patch.diff
+  cd ..
+  cd ..
+
   cmake --version
 
   cmake -S dep-adios2/ADIOS2-2.10.2 -B build-adios2 ^
@@ -87,7 +103,8 @@ exit /b 0
     -DBUILD_TESTS=OFF           ^
     -DPIGZ_ENABLE_TESTS=OFF     ^
     -DZLIB_ENABLE_TESTS=OFF     ^
-    -DZLIBNG_ENABLE_TESTS=OFF
+    -DZLIBNG_ENABLE_TESTS=OFF   ^
+    -DDEACTIVATE_AVX512=ON
 ::    -DPREFER_EXTERNAL_ZLIB=ON   ^
 ::    -DZLIB_USE_STATIC_LIBS=ON
   if errorlevel 1 exit 1
@@ -146,10 +163,10 @@ exit /b 0
 :build_zfp
   if exist zfp-stamp exit /b 0
 
-  curl -sLo zfp-0.5.5.tar.gz ^
-    https://github.com/LLNL/zfp/releases/download/0.5.5/zfp-0.5.5.tar.gz
-  tar -xvzf zfp-0.5.5.tar.gz
-  mv zfp-0.5.5 dep-zfp
+  curl -sLo zfp-1.0.1.tar.gz ^
+    https://github.com/LLNL/zfp/releases/download/1.0.1/zfp-1.0.1.tar.gz
+  tar -xvzf zfp-1.0.1.tar.gz
+  mv zfp-1.0.1 dep-zfp
 
   cmake -S dep-zfp -B build-zfp ^
     -DCMAKE_BUILD_TYPE=Release  ^
@@ -176,11 +193,11 @@ exit /b 0
 :build_zlib
   if exist zlib-stamp exit /b 0
 
-  curl -sLo zlib-1.2.13.zip ^
-    https://github.com/madler/zlib/archive/v1.2.13.zip
-  powershell Expand-Archive zlib-1.2.13.zip -DestinationPath dep-zlib
+  curl -sLo zlib-1.3.1.zip ^
+    https://github.com/madler/zlib/archive/v1.3.1.zip
+  powershell Expand-Archive zlib-1.3.1.zip -DestinationPath dep-zlib
 
-  cmake -S dep-zlib/zlib-1.2.13 -B build-zlib ^
+  cmake -S dep-zlib/zlib-1.3.1 -B build-zlib ^
     -DBUILD_SHARED_LIBS=ON ^
     -DCMAKE_BUILD_TYPE=Release
   if errorlevel 1 exit 1

library_builders.sh

@@ -72,34 +72,6 @@ function install_buildessentials {
     touch buildessentials-stamp
 }
 
-function build_adios1 {
-    if [ -e adios1-stamp ]; then return; fi
-
-    curl -k -sLo adios-1.13.1.tar.gz \
-        https://users.nccs.gov/~pnorbert/adios-1.13.1.tar.gz
-    file adios*.tar.gz
-    tar -xzf adios*.tar.gz
-    rm adios*.tar.gz
-    cd adios-*
-
-    # Cross-Compile hints for autotools based builds
-    HOST_ARG=""
-    if [[ "${CMAKE_OSX_ARCHITECTURES-}" == "arm64" ]]; then
-        HOST_ARG="--host=aarch64-apple-darwin"
-    fi
-
-    ./configure --enable-static --disable-shared --disable-fortran --without-mpi ${HOST_ARG} --prefix=${BUILD_PREFIX} --with-blosc=/usr
-    make -j${CPU_COUNT}
-    make install
-    cd -
-
-    # note: for universal binaries on macOS
-    #   https://developer.apple.com/documentation/apple-silicon/building-a-universal-macos-binary
-    #lipo -create -output universal_app x86_app arm_app
-
-    touch adios1-stamp
-}
-
 function build_adios2 {
     if [ -e adios2-stamp ]; then return; fi
 
@@ -153,51 +125,6 @@ function build_adios2 {
     touch adios2-stamp
 }
 
-function build_blosc {
-    if [ -e blosc-stamp ]; then return; fi
-
-    curl -sLo c-blosc-1.21.0.tar.gz \
-        https://github.com/Blosc/c-blosc/archive/v1.21.0.tar.gz
-    file c-blosc*.tar.gz
-    tar -xzf c-blosc*.tar.gz
-    rm c-blosc*.tar.gz
-
-    # Patch PThread Propagation
-    curl -sLo blosc-pthread.patch \
-        https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc/pull/318.patch
-    python3 -m patch -p 1 -d c-blosc-1.21.0 blosc-pthread.patch
-
-    # SSE2 support
-    #   https://github.com/Blosc/c-blosc/issues/334
-    DEACTIVATE_SSE2=OFF
-    if [[ "${CMAKE_OSX_ARCHITECTURES-}" == *"arm64"* ]]; then
-      # error: SSE2 is not supported by the target architecture/platform and/or this compiler.
-      DEACTIVATE_SSE2=ON
-    fi
-
-    mkdir build-blosc
-    cd build-blosc
-    PY_BIN=$(which python3)
-    CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/"
-    PATH=${CMAKE_BIN}:${PATH} cmake          \
-      -DDEACTIVATE_SNAPPY=ON                 \
-      -DDEACTIVATE_SSE2=${DEACTIVATE_SSE2}   \
-      -DBUILD_SHARED=OFF                     \
-      -DBUILD_TESTS=OFF                      \
-      -DBUILD_BENCHMARKS=OFF                 \
-      -DCMAKE_VERBOSE_MAKEFILE=ON            \
-      -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} \
-      -DZLIB_USE_STATIC_LIBS=ON              \
-      ../c-blosc-*
-    make -j${CPU_COUNT}
-    make install
-    cd -
-
-    rm -rf build-blosc
-
-    touch blosc-stamp
-}
-
 function build_blosc2 {
     if [ -e blosc-stamp2 ]; then return; fi
 
@@ -209,6 +136,15 @@ function build_blosc2 {
 
     mkdir build-blosc2
     cd build-blosc2
+    if [[ "${CMAKE_OSX_ARCHITECTURES-}" == *"arm64"* ]]; then
+        # SSE2 support
+        #   https://github.com/Blosc/c-blosc/issues/334
+        # error: SSE2 is not supported by the target architecture/platform and/or this compiler.
+        local architecture_specific_flags=("-DDEACTIVATE_SSE2=ON")
+    else
+        # AVX512 not supported on AMD CPUs
+        local architecture_specific_flags=("-DDEACTIVATE_SSE2=OFF" "-DDEACTIVATE_AVX512=ON")
+    fi
     PY_BIN=$(which python3)
     CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/"
     PATH=${CMAKE_BIN}:${PATH} cmake          \
@@ -224,6 +160,7 @@ function build_blosc2 {
       -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} \
       -DPREFER_EXTERNAL_ZLIB=ON              \
       -DZLIB_USE_STATIC_LIBS=ON              \
+      "${architecture_specific_flags[@]}"    \
       ../c-blosc2-*
     make -j${CPU_COUNT}
     make install
@@ -237,8 +174,9 @@ function build_blosc2 {
 function build_zfp {
     if [ -e zfp-stamp ]; then return; fi
 
-    curl -sLo zfp-0.5.5.tar.gz \
-        https://github.com/LLNL/zfp/releases/download/0.5.5/zfp-0.5.5.tar.gz
+    local version="1.0.1"
+    curl -sLo zfp-$version.tar.gz \
+        https://github.com/LLNL/zfp/releases/download/$version/zfp-$version.tar.gz
     file zfp*.tar.gz
     tar -xzf zfp*.tar.gz
     rm zfp*.tar.gz
@@ -265,7 +203,7 @@ function build_zfp {
 function build_zlib {
     if [ -e zlib-stamp ]; then return; fi
 
-    ZLIB_VERSION="1.2.13"
+    ZLIB_VERSION="1.3.1"
 
     curl -sLO https://zlib.net/fossils/zlib-$ZLIB_VERSION.tar.gz
     file zlib*.tar.gz
@@ -377,15 +315,6 @@ fi
 install_buildessentials
 build_zlib
 build_zfp
-if [[ "$(uname -m)" != "ppc64le" ]]; then
-    # builds too long for Travis-CI
-    build_blosc
-fi
 build_blosc2
 build_hdf5
-if [[ "${CMAKE_OSX_ARCHITECTURES-}" != "arm64" && "$(uname -m)" != "ppc64le" ]]; then
-    # macOS: skip ADIOS1 build for M1
-    # Linux: with ADIOS2 also enabled, this builds too long for Travis-CI
-    build_adios1
-fi
 build_adios2