From 0520e94a6894420214e0273f568cd423e1546c11 Mon Sep 17 00:00:00 2001 From: Kevin Squire Date: Fri, 17 Sep 2021 13:17:19 -0700 Subject: [PATCH] Update MKL recipe (#3641) * [MKL] Reorganize script to so that selecting individual platforms works * MKL is one of the few packages with a different source archive for each architecture. Previously, the script would download and generate packages for multiple platforms even if they were not selected. * Here, we associate each ArchiveSource and other build parameters with each platform and filter the install list appropriately. * [MKL] Deduplicate macOS dylibs The macOS archive we are currently installing contains byte-identical .dylib files (e.g., libmkl_core.dylib and libmkl_core.1.dylib), which take up a lot of space. Normally, these would be symlinks (or possibly hard links). Until these are fixed upstream, we deduplicate them here when building the tarballs for macOS. * [MKL] Ensure "--register" is passed only on the last invocation of build_tarballs() --- M/MKL/build_tarballs.jl | 113 +++++++++++++++++++++++++--------------- M/MKL/script.sh | 8 +++ M/MKL/script_macos.sh | 38 ++++++++++++++ 3 files changed, 117 insertions(+), 42 deletions(-) create mode 100644 M/MKL/script.sh create mode 100644 M/MKL/script_macos.sh diff --git a/M/MKL/build_tarballs.jl b/M/MKL/build_tarballs.jl index 1d84a86bfbb..899af5ed1ec 100644 --- a/M/MKL/build_tarballs.jl +++ b/M/MKL/build_tarballs.jl @@ -3,37 +3,69 @@ using BinaryBuilder, Pkg name = "MKL" version = v"2021.1.1" -sources = [ - ArchiveSource("https://anaconda.org/intel/mkl/2021.1.1/download/linux-64/mkl-2021.1.1-intel_52.tar.bz2", - "bfb0fd056576cad99ae1d9c69ada2745420da9f9cf052551d5b91f797538bda2"; unpack_target = "mkl-x86_64-linux-gnu"), - ArchiveSource("https://anaconda.org/intel/mkl/2021.1.1/download/linux-32/mkl-2021.1.1-intel_52.tar.bz2", - "7b6f55a30886154bd96d4b4c6b7428494a59397b87779b58e5b3de00250343f9"; unpack_target = "mkl-i686-linux-gnu"), - ArchiveSource("https://anaconda.org/intel/mkl/2021.1.1/download/osx-64/mkl-2021.1.1-intel_50.tar.bz2", - "819fb8875909d4d024e2a936c54b561aebd1e3aebe58fc605c70aa1ad9a66b70"; unpack_target = "mkl-x86_64-apple-darwin14"), - ArchiveSource("https://anaconda.org/intel/mkl/2021.1.1/download/win-32/mkl-2021.1.1-intel_52.tar.bz2", - "dba6a12a481407ec55fba9895b68afacb15f044905dcb5e185db341b688e6177"; unpack_target = "mkl-i686-w64-mingw32"), - ArchiveSource("https://anaconda.org/intel/mkl/2021.1.1/download/win-64/mkl-2021.1.1-intel_52.tar.bz2", - "4024391b8a45836d5a7ee92405b7767874b3c3bbf2f490349fda042db3b60dfd"; unpack_target = "mkl-x86_64-w64-mingw32"), -] - -# Bash recipe for building across all platforms -script = raw""" -cd ${WORKSPACE}/srcdir/mkl-${target} -if [[ ${target} == *-mingw* ]]; then - cp -r Library/bin/* ${libdir} -else - cp -r lib/* ${libdir} -fi +# Bash recipes for building across all platforms +script = read(joinpath(@__DIR__, "script.sh"), String) +script_macos = read(joinpath(@__DIR__, "script_macos.sh"), String) -install_license info/licenses/*.txt -""" +non_reg_ARGS = filter(arg -> arg != "--register", ARGS) -platforms = [ - Platform("x86_64", "linux"; libc="glibc"), - Platform("i686", "linux"; libc="glibc"), - Platform("x86_64", "macos"), - Platform("i686", "windows"), - Platform("x86_64", "windows"), +platform_sources = [ + ( + platform = Platform("x86_64", "linux"; libc="glibc"), + source = ArchiveSource( + "https://anaconda.org/intel/mkl/2021.1.1/download/linux-64/mkl-2021.1.1-intel_52.tar.bz2", + "bfb0fd056576cad99ae1d9c69ada2745420da9f9cf052551d5b91f797538bda2"; + unpack_target = "mkl-x86_64-linux-gnu" + ), + # We need to run autofix on Linux, because here libmkl_rt doesn't + # have a soname, so we can't ccall it without specifying the path: + # https://github.com/JuliaSparse/Pardiso.jl/issues/69 + autofix = true, + script = script, + ), + ( + platform = Platform("i686", "linux"; libc="glibc"), + source = ArchiveSource( + "https://anaconda.org/intel/mkl/2021.1.1/download/linux-32/mkl-2021.1.1-intel_52.tar.bz2", + "7b6f55a30886154bd96d4b4c6b7428494a59397b87779b58e5b3de00250343f9"; + unpack_target = "mkl-i686-linux-gnu" + ), + autofix = true, + script = script, + ), + ( + platform = Platform("x86_64", "macos"), + source = ArchiveSource( + "https://anaconda.org/intel/mkl/2021.1.1/download/osx-64/mkl-2021.1.1-intel_50.tar.bz2", + "819fb8875909d4d024e2a936c54b561aebd1e3aebe58fc605c70aa1ad9a66b70"; + unpack_target = "mkl-x86_64-apple-darwin14" + ), + # Need to disable autofix: updating linkage of libmkl_intel_thread.dylib on + # macOS causes runtime issues: + # https://github.com/JuliaPackaging/Yggdrasil/issues/915. + autofix = false, + script = script_macos, + ), + ( + platform = Platform("i686", "windows"), + source = ArchiveSource( + "https://anaconda.org/intel/mkl/2021.1.1/download/win-32/mkl-2021.1.1-intel_52.tar.bz2", + "dba6a12a481407ec55fba9895b68afacb15f044905dcb5e185db341b688e6177"; + unpack_target = "mkl-i686-w64-mingw32" + ), + autofix = false, + script = script, + ), + ( + platform = Platform("x86_64", "windows"), + source = ArchiveSource( + "https://anaconda.org/intel/mkl/2021.1.1/download/win-64/mkl-2021.1.1-intel_52.tar.bz2", + "4024391b8a45836d5a7ee92405b7767874b3c3bbf2f490349fda042db3b60dfd"; + unpack_target = "mkl-x86_64-w64-mingw32" + ), + autofix = false, + script = script, + ), ] # The products that we will ensure are always built @@ -49,17 +81,14 @@ dependencies = [ non_reg_ARGS = filter(arg -> arg != "--register", ARGS) include("../../fancy_toys.jl") -no_autofix_platforms = [Platform("i686", "windows"), Platform("x86_64", "windows"), Platform("x86_64", "macos")] -autofix_platforms = [Platform("x86_64", "linux"), Platform("i686", "linux")] -if any(should_build_platform.(triplet.(no_autofix_platforms))) - # Need to disable autofix: updating linkage of libmkl_intel_thread.dylib on - # macOS causes runtime issues: - # https://github.com/JuliaPackaging/Yggdrasil/issues/915. - build_tarballs(non_reg_ARGS, name, version, sources, script, no_autofix_platforms, products, dependencies; lazy_artifacts = true, autofix = false) -end -if any(should_build_platform.(triplet.(autofix_platforms))) - # ... but we need to run autofix on Linux, because here libmkl_rt doesn't - # have a soname, so we can't ccall it without specifying the path: - # https://github.com/JuliaSparse/Pardiso.jl/issues/69 - build_tarballs(ARGS, name, version, sources, script, autofix_platforms, products, dependencies; lazy_artifacts = true) +filter!(p -> should_build_platform(triplet(first(p))), platform_sources) + +for (idx, (platform, source, autofix, script)) in enumerate(platform_sources) + # Use "--register" only on the last invocation of build_tarballs + if idx < length(platform_sources) + args = non_reg_ARGS + else + args = ARGS + end + build_tarballs(args, name, version, [source], script, [platform], products, dependencies; lazy_artifacts = true, autofix = autofix) end diff --git a/M/MKL/script.sh b/M/MKL/script.sh new file mode 100644 index 00000000000..e9dc6a6f153 --- /dev/null +++ b/M/MKL/script.sh @@ -0,0 +1,8 @@ +cd ${WORKSPACE}/srcdir/mkl-${target} +if [[ ${target} == *-mingw* ]]; then + cp -r Library/bin/* ${libdir} +else + cp -r lib/* ${libdir} +fi + +install_license info/licenses/*.txt \ No newline at end of file diff --git a/M/MKL/script_macos.sh b/M/MKL/script_macos.sh new file mode 100644 index 00000000000..7f426c0c279 --- /dev/null +++ b/M/MKL/script_macos.sh @@ -0,0 +1,38 @@ + +sha256() { sha256sum "$1" | awk '{ print $1 }'; } + +exists() { + if [ "$2" != in ]; then + echo "Incorrect usage. Use: exists {key} in {array}" + return + fi + eval '[ ${'$3'[$1]+x} ]' +} + +cd ${WORKSPACE}/srcdir/mkl-${target}/lib + +##### +## Replace duplicate library files with symlinks +## (https://github.com/JuliaPackaging/Yggdrasil/issues/3632) + +declare -A SHAs + +# Iterate over files in reverse order of name length. +# This ensures any symlinks go to the longer name. +for file in $(find . -type f | awk '{ print length, $0 }' | sort -n -r | cut -d" " -f2-); do + sha=$(sha256 "$file") + + if ! exists $sha in SHAs; then + SHAs[$sha]="$file" + continue + fi + + src_file="${SHAs[$sha]}" + ln -sf $src_file $file +done + +##### + +cd .. +cp -r lib/* ${libdir} +install_license info/licenses/*.txt \ No newline at end of file