Skip to content

Commit

Permalink
Update MKL recipe (JuliaPackaging#3641)
Browse files Browse the repository at this point in the history
* [MKL] Reorganize script to so that selecting individual platforms works

* MKL is one of the few packages with a different source archive for each
  architecture.  Previously, the script would download and generate
  packages for multiple platforms even if they were not selected.
* Here, we associate each ArchiveSource and other build parameters with
  each platform and filter the install list appropriately.

* [MKL] Deduplicate macOS dylibs

The macOS archive we are currently installing contains byte-identical
.dylib files (e.g., libmkl_core.dylib and libmkl_core.1.dylib), which
take up a lot of space.  Normally,  these would be symlinks (or possibly
hard links).  Until these are fixed upstream, we deduplicate them here
when building the tarballs for macOS.

* [MKL] Ensure "--register" is passed only on the last invocation of build_tarballs()
  • Loading branch information
kmsquire authored Sep 17, 2021
1 parent 8d6c4cd commit 0520e94
Show file tree
Hide file tree
Showing 3 changed files with 117 additions and 42 deletions.
113 changes: 71 additions & 42 deletions M/MKL/build_tarballs.jl
Original file line number Diff line number Diff line change
Expand Up @@ -3,37 +3,69 @@ using BinaryBuilder, Pkg
name = "MKL"
version = v"2021.1.1"

sources = [
ArchiveSource("https://anaconda.org/intel/mkl/2021.1.1/download/linux-64/mkl-2021.1.1-intel_52.tar.bz2",
"bfb0fd056576cad99ae1d9c69ada2745420da9f9cf052551d5b91f797538bda2"; unpack_target = "mkl-x86_64-linux-gnu"),
ArchiveSource("https://anaconda.org/intel/mkl/2021.1.1/download/linux-32/mkl-2021.1.1-intel_52.tar.bz2",
"7b6f55a30886154bd96d4b4c6b7428494a59397b87779b58e5b3de00250343f9"; unpack_target = "mkl-i686-linux-gnu"),
ArchiveSource("https://anaconda.org/intel/mkl/2021.1.1/download/osx-64/mkl-2021.1.1-intel_50.tar.bz2",
"819fb8875909d4d024e2a936c54b561aebd1e3aebe58fc605c70aa1ad9a66b70"; unpack_target = "mkl-x86_64-apple-darwin14"),
ArchiveSource("https://anaconda.org/intel/mkl/2021.1.1/download/win-32/mkl-2021.1.1-intel_52.tar.bz2",
"dba6a12a481407ec55fba9895b68afacb15f044905dcb5e185db341b688e6177"; unpack_target = "mkl-i686-w64-mingw32"),
ArchiveSource("https://anaconda.org/intel/mkl/2021.1.1/download/win-64/mkl-2021.1.1-intel_52.tar.bz2",
"4024391b8a45836d5a7ee92405b7767874b3c3bbf2f490349fda042db3b60dfd"; unpack_target = "mkl-x86_64-w64-mingw32"),
]

# Bash recipe for building across all platforms
script = raw"""
cd ${WORKSPACE}/srcdir/mkl-${target}
if [[ ${target} == *-mingw* ]]; then
cp -r Library/bin/* ${libdir}
else
cp -r lib/* ${libdir}
fi
# Bash recipes for building across all platforms
script = read(joinpath(@__DIR__, "script.sh"), String)
script_macos = read(joinpath(@__DIR__, "script_macos.sh"), String)

install_license info/licenses/*.txt
"""
non_reg_ARGS = filter(arg -> arg != "--register", ARGS)

platforms = [
Platform("x86_64", "linux"; libc="glibc"),
Platform("i686", "linux"; libc="glibc"),
Platform("x86_64", "macos"),
Platform("i686", "windows"),
Platform("x86_64", "windows"),
platform_sources = [
(
platform = Platform("x86_64", "linux"; libc="glibc"),
source = ArchiveSource(
"https://anaconda.org/intel/mkl/2021.1.1/download/linux-64/mkl-2021.1.1-intel_52.tar.bz2",
"bfb0fd056576cad99ae1d9c69ada2745420da9f9cf052551d5b91f797538bda2";
unpack_target = "mkl-x86_64-linux-gnu"
),
# We need to run autofix on Linux, because here libmkl_rt doesn't
# have a soname, so we can't ccall it without specifying the path:
# https://github.com/JuliaSparse/Pardiso.jl/issues/69
autofix = true,
script = script,
),
(
platform = Platform("i686", "linux"; libc="glibc"),
source = ArchiveSource(
"https://anaconda.org/intel/mkl/2021.1.1/download/linux-32/mkl-2021.1.1-intel_52.tar.bz2",
"7b6f55a30886154bd96d4b4c6b7428494a59397b87779b58e5b3de00250343f9";
unpack_target = "mkl-i686-linux-gnu"
),
autofix = true,
script = script,
),
(
platform = Platform("x86_64", "macos"),
source = ArchiveSource(
"https://anaconda.org/intel/mkl/2021.1.1/download/osx-64/mkl-2021.1.1-intel_50.tar.bz2",
"819fb8875909d4d024e2a936c54b561aebd1e3aebe58fc605c70aa1ad9a66b70";
unpack_target = "mkl-x86_64-apple-darwin14"
),
# Need to disable autofix: updating linkage of libmkl_intel_thread.dylib on
# macOS causes runtime issues:
# https://github.com/JuliaPackaging/Yggdrasil/issues/915.
autofix = false,
script = script_macos,
),
(
platform = Platform("i686", "windows"),
source = ArchiveSource(
"https://anaconda.org/intel/mkl/2021.1.1/download/win-32/mkl-2021.1.1-intel_52.tar.bz2",
"dba6a12a481407ec55fba9895b68afacb15f044905dcb5e185db341b688e6177";
unpack_target = "mkl-i686-w64-mingw32"
),
autofix = false,
script = script,
),
(
platform = Platform("x86_64", "windows"),
source = ArchiveSource(
"https://anaconda.org/intel/mkl/2021.1.1/download/win-64/mkl-2021.1.1-intel_52.tar.bz2",
"4024391b8a45836d5a7ee92405b7767874b3c3bbf2f490349fda042db3b60dfd";
unpack_target = "mkl-x86_64-w64-mingw32"
),
autofix = false,
script = script,
),
]

# The products that we will ensure are always built
Expand All @@ -49,17 +81,14 @@ dependencies = [

non_reg_ARGS = filter(arg -> arg != "--register", ARGS)
include("../../fancy_toys.jl")
no_autofix_platforms = [Platform("i686", "windows"), Platform("x86_64", "windows"), Platform("x86_64", "macos")]
autofix_platforms = [Platform("x86_64", "linux"), Platform("i686", "linux")]
if any(should_build_platform.(triplet.(no_autofix_platforms)))
# Need to disable autofix: updating linkage of libmkl_intel_thread.dylib on
# macOS causes runtime issues:
# https://github.com/JuliaPackaging/Yggdrasil/issues/915.
build_tarballs(non_reg_ARGS, name, version, sources, script, no_autofix_platforms, products, dependencies; lazy_artifacts = true, autofix = false)
end
if any(should_build_platform.(triplet.(autofix_platforms)))
# ... but we need to run autofix on Linux, because here libmkl_rt doesn't
# have a soname, so we can't ccall it without specifying the path:
# https://github.com/JuliaSparse/Pardiso.jl/issues/69
build_tarballs(ARGS, name, version, sources, script, autofix_platforms, products, dependencies; lazy_artifacts = true)
filter!(p -> should_build_platform(triplet(first(p))), platform_sources)

for (idx, (platform, source, autofix, script)) in enumerate(platform_sources)
# Use "--register" only on the last invocation of build_tarballs
if idx < length(platform_sources)
args = non_reg_ARGS
else
args = ARGS
end
build_tarballs(args, name, version, [source], script, [platform], products, dependencies; lazy_artifacts = true, autofix = autofix)
end
8 changes: 8 additions & 0 deletions M/MKL/script.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
cd ${WORKSPACE}/srcdir/mkl-${target}
if [[ ${target} == *-mingw* ]]; then
cp -r Library/bin/* ${libdir}
else
cp -r lib/* ${libdir}
fi

install_license info/licenses/*.txt
38 changes: 38 additions & 0 deletions M/MKL/script_macos.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@

sha256() { sha256sum "$1" | awk '{ print $1 }'; }

exists() {
if [ "$2" != in ]; then
echo "Incorrect usage. Use: exists {key} in {array}"
return
fi
eval '[ ${'$3'[$1]+x} ]'
}

cd ${WORKSPACE}/srcdir/mkl-${target}/lib

#####
## Replace duplicate library files with symlinks
## (https://github.com/JuliaPackaging/Yggdrasil/issues/3632)

declare -A SHAs

# Iterate over files in reverse order of name length.
# This ensures any symlinks go to the longer name.
for file in $(find . -type f | awk '{ print length, $0 }' | sort -n -r | cut -d" " -f2-); do
sha=$(sha256 "$file")

if ! exists $sha in SHAs; then
SHAs[$sha]="$file"
continue
fi

src_file="${SHAs[$sha]}"
ln -sf $src_file $file
done

#####

cd ..
cp -r lib/* ${libdir}
install_license info/licenses/*.txt

0 comments on commit 0520e94

Please sign in to comment.