From 20911b22dbb28f0b28b0c460f869b231797d9c0c Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Mon, 7 Apr 2025 16:17:08 +0200 Subject: [PATCH 01/32] [SVS] Fix CI builds * Add CPU detection for SVS * Add MKL installation for SVS shared library --- .install/amazon_linux_2.sh | 1 + .install/common_base_linux_mariner_2.0.sh | 1 + .install/debian_gnu_linux_11.sh | 1 + .install/install_mkl.sh | 19 +++++++++++++++++++ .install/rocky_linux_8.sh | 1 + .install/rocky_linux_9.sh | 1 + .install/ubuntu_16.04.sh | 1 + .install/ubuntu_18.04.sh | 1 + .install/ubuntu_20.04.sh | 1 + .install/ubuntu_22.04.sh | 1 + .install/ubuntu_24.04.sh | 1 + src/VecSim/CMakeLists.txt | 22 +++++++++++++++++----- tests/unit/CMakeLists.txt | 2 +- 13 files changed, 47 insertions(+), 6 deletions(-) create mode 100644 .install/install_mkl.sh diff --git a/.install/amazon_linux_2.sh b/.install/amazon_linux_2.sh index 06ed36887..9d1b55d08 100644 --- a/.install/amazon_linux_2.sh +++ b/.install/amazon_linux_2.sh @@ -10,3 +10,4 @@ $MODE yum install -y wget git valgrind gcc10 gcc10-c++ $MODE update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc10-gcc 60 --slave /usr/bin/g++ g++ /usr/bin/gcc10-g++ source install_cmake.sh $MODE +source install_mkl.sh $MODE diff --git a/.install/common_base_linux_mariner_2.0.sh b/.install/common_base_linux_mariner_2.0.sh index e928e7e74..82033013c 100644 --- a/.install/common_base_linux_mariner_2.0.sh +++ b/.install/common_base_linux_mariner_2.0.sh @@ -6,3 +6,4 @@ export DEBIAN_FRONTEND=noninteractive $MODE tdnf install -y build-essential git wget ca-certificates which source install_cmake.sh $MODE +source install_mkl.sh $MODE diff --git a/.install/debian_gnu_linux_11.sh b/.install/debian_gnu_linux_11.sh index febfb4a44..f0d741995 100644 --- a/.install/debian_gnu_linux_11.sh +++ b/.install/debian_gnu_linux_11.sh @@ -6,3 +6,4 @@ MODE=$1 # whether to install using sudo or not $MODE apt update -qq $MODE apt install -yqq git wget build-essential lcov valgrind source install_cmake.sh $MODE +source install_mkl.sh $MODE diff --git a/.install/install_mkl.sh b/.install/install_mkl.sh new file mode 100644 index 000000000..9dc943186 --- /dev/null +++ b/.install/install_mkl.sh @@ -0,0 +1,19 @@ +#!/bin/bash +version=2025.1.0.803 +uuid=dc93af13-2b3f-40c3-a41b-2bc05a707a80 +prefix=/opt/intel/oneapi +processor=$(uname -m) +OS_TYPE=$(uname -s) +MODE=$1 # whether to install using sudo or not + +if [[ $OS_TYPE = 'Linux' ]] +then + if [[ $processor = 'x86_64' ]] + then + filename=intel-onemkl-${version}_offline.sh + wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/${uuid}/${filename} + chmod u+x ./${filename} + $MODE ./${filename} -a --silent --eula accept --install-dir ${prefix} + source ${prefix}/setvars.sh + fi +fi diff --git a/.install/rocky_linux_8.sh b/.install/rocky_linux_8.sh index dbb6cdbb9..1a5db2143 100755 --- a/.install/rocky_linux_8.sh +++ b/.install/rocky_linux_8.sh @@ -19,3 +19,4 @@ $MODE dnf install -y gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ gcc-toolset-13-li cp /opt/rh/gcc-toolset-13/enable /etc/profile.d/gcc-toolset-13.sh source install_cmake.sh $MODE +source install_mkl.sh $MODE diff --git a/.install/rocky_linux_9.sh b/.install/rocky_linux_9.sh index 523998c8c..05f83fef5 100644 --- a/.install/rocky_linux_9.sh +++ b/.install/rocky_linux_9.sh @@ -8,3 +8,4 @@ $MODE dnf install -y gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ make wget git val cp /opt/rh/gcc-toolset-13/enable /etc/profile.d/gcc-toolset-13.sh source install_cmake.sh $MODE +source install_mkl.sh $MODE diff --git a/.install/ubuntu_16.04.sh b/.install/ubuntu_16.04.sh index 2eef88bb7..9cec6ee52 100755 --- a/.install/ubuntu_16.04.sh +++ b/.install/ubuntu_16.04.sh @@ -12,3 +12,4 @@ $MODE apt update $MODE apt-get install -yqq git wget make valgrind gcc-9 g++-9 $MODE update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-9 source install_cmake.sh $MODE +source install_mkl.sh $MODE diff --git a/.install/ubuntu_18.04.sh b/.install/ubuntu_18.04.sh index 73390b401..d1c812342 100644 --- a/.install/ubuntu_18.04.sh +++ b/.install/ubuntu_18.04.sh @@ -21,3 +21,4 @@ $MODE make install cd .. source install_cmake.sh $MODE +source install_mkl.sh $MODE diff --git a/.install/ubuntu_20.04.sh b/.install/ubuntu_20.04.sh index 6b45d2fe8..08282ccac 100755 --- a/.install/ubuntu_20.04.sh +++ b/.install/ubuntu_20.04.sh @@ -12,3 +12,4 @@ $MODE update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 60 --slave # align gcov version with gcc version $MODE update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-11 60 source install_cmake.sh $MODE +source install_mkl.sh $MODE diff --git a/.install/ubuntu_22.04.sh b/.install/ubuntu_22.04.sh index 0075f9a1e..1a91daf12 100755 --- a/.install/ubuntu_22.04.sh +++ b/.install/ubuntu_22.04.sh @@ -9,3 +9,4 @@ $MODE update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 60 --slave # align gcov version with gcc version $MODE update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-12 60 source install_cmake.sh $MODE +source install_mkl.sh $MODE diff --git a/.install/ubuntu_24.04.sh b/.install/ubuntu_24.04.sh index 4c06a346a..ee96eb00a 100755 --- a/.install/ubuntu_24.04.sh +++ b/.install/ubuntu_24.04.sh @@ -7,3 +7,4 @@ $MODE apt update -qq $MODE apt install -yqq git wget build-essential lcov openssl libssl-dev \ python3 python3-venv python3-dev unzip rsync clang curl source install_cmake.sh $MODE +source install_mkl.sh $MODE diff --git a/src/VecSim/CMakeLists.txt b/src/VecSim/CMakeLists.txt index 58d45f203..657fb302a 100644 --- a/src/VecSim/CMakeLists.txt +++ b/src/VecSim/CMakeLists.txt @@ -22,15 +22,27 @@ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g") # add_definitions(-DSVS_EXPERIMENTAL_LEANVEC) # endif() -# TODO: enable svs build again once MKL installation issue is resolved, and after we validate -# that we only build for supported platforms (and fail gracefully for those that are not) -option(SVS_SUPPORTED "Build with SVS library support" OFF) +if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "(x86_64)|(AMD64|amd64)") + set(SVS_SUPPORTED 1) +else() + set(SVS_SUPPORTED 0) + message(STATUS "SVS is not supported on this architecture") +endif() + +# USE_SVS option forsibly OFF if CPU is not supported +# elsewhere let user disable SVS +cmake_dependent_option(USE_SVS "Build with SVS library support" ON "SVS_SUPPORTED" OFF) +if(USE_SVS) + message(STATUS "SVS support enabled") +else() + message(STATUS "SVS support disabled") +endif() # Configure SVS build set(SVS_SHARED_LIB ON CACHE BOOL "Use SVS pre-compiled shared library") set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.7/svs-shared-library-0.0.7-avx2.tar.gz" CACHE STRING "SVS URL") -if(SVS_SUPPORTED) +if(USE_SVS) set(svs_factory_file "index_factories/svs_factory.cpp") if(SVS_SHARED_LIB) include(FetchContent) @@ -78,7 +90,7 @@ add_library(VectorSimilarity ${VECSIM_LIBTYPE} ${HEADER_LIST} ) -if (SVS_SUPPORTED) +if (USE_SVS) target_link_libraries(VectorSimilarity VectorSimilaritySpaces svs::svs) if(TARGET svs::svs_shared_library) target_link_libraries(VectorSimilarity svs::svs_shared_library MKL::MKL) diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index f248705d4..67a281868 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -75,7 +75,7 @@ gtest_discover_tests(test_fp16 TEST_PREFIX FP16UNIT_) gtest_discover_tests(test_int8 TEST_PREFIX INT8UNIT_) gtest_discover_tests(test_uint8 TEST_PREFIX UINT8UNIT_) -if(SVS_SUPPORTED) +if(USE_SVS) add_executable(test_svs ../utils/mock_thread_pool.cpp test_svs.cpp unit_test_utils.cpp) target_link_libraries(test_svs PUBLIC gtest_main VectorSimilarity) gtest_discover_tests(test_svs) From dec1b2107de72b0a21b69498856c6ca2a0b4e8fc Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Mon, 7 Apr 2025 16:48:25 +0200 Subject: [PATCH 02/32] Extend MKL package lookup in CMAKE --- src/VecSim/CMakeLists.txt | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/VecSim/CMakeLists.txt b/src/VecSim/CMakeLists.txt index 657fb302a..cd2b9b202 100644 --- a/src/VecSim/CMakeLists.txt +++ b/src/VecSim/CMakeLists.txt @@ -39,9 +39,22 @@ else() endif() # Configure SVS build -set(SVS_SHARED_LIB ON CACHE BOOL "Use SVS pre-compiled shared library") +set(SVS_SHARED_LIB OFF CACHE BOOL "Use SVS pre-compiled shared library") set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.7/svs-shared-library-0.0.7-avx2.tar.gz" CACHE STRING "SVS URL") +set(MKL_HINTS + ${MKLROOT} + $ENV{MKL_DIR} + $ENV{MKL_ROOT} + $ENV{MKLROOT}) + +if(LINUX) + list(APPEND MKL_HINTS "/opt/intel/oneapi/mkl/latest") +endif() +if(WIN32) + list(APPEND MKL_HINTS "C:/Program Files (x86)/Intel/oneAPI/mkl/latest") +endif() + if(USE_SVS) set(svs_factory_file "index_factories/svs_factory.cpp") if(SVS_SHARED_LIB) @@ -52,7 +65,7 @@ if(USE_SVS) ) FetchContent_MakeAvailable(svs) list(APPEND CMAKE_PREFIX_PATH "${svs_SOURCE_DIR}") - find_package(MKL REQUIRED) + find_package(MKL REQUIRED HINTS ${MKL_HINTS}) find_package(svs REQUIRED) set(SVS_LVQ_HEADER "svs/extensions/vamana/lvq.h") else() From 63502e95cd7b65055ce4cdbbf80ec5bfd1e360ea Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Tue, 8 Apr 2025 14:54:13 +0200 Subject: [PATCH 03/32] Change MKL version to 2024.1 --- .install/install_mkl.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.install/install_mkl.sh b/.install/install_mkl.sh index 9dc943186..617ef36a7 100644 --- a/.install/install_mkl.sh +++ b/.install/install_mkl.sh @@ -1,6 +1,6 @@ #!/bin/bash -version=2025.1.0.803 -uuid=dc93af13-2b3f-40c3-a41b-2bc05a707a80 +filename=l_BaseKit_p_2024.1.0.596.sh +uuid=fdc7a2bc-b7a8-47eb-8876-de6201297144 prefix=/opt/intel/oneapi processor=$(uname -m) OS_TYPE=$(uname -s) @@ -10,10 +10,9 @@ if [[ $OS_TYPE = 'Linux' ]] then if [[ $processor = 'x86_64' ]] then - filename=intel-onemkl-${version}_offline.sh wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/${uuid}/${filename} chmod u+x ./${filename} - $MODE ./${filename} -a --silent --eula accept --install-dir ${prefix} - source ${prefix}/setvars.sh + $MODE ./${filename} -a --action install --silent --eula accept --components intel.oneapi.lin.mkl.devel --install-dir ${prefix} + #source ${prefix}/setvars.sh fi fi From a31a1c8d52bf1e3cfa7f8f3341c2b11a0bf3d573 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Tue, 8 Apr 2025 15:25:36 +0200 Subject: [PATCH 04/32] Add --ignore-errors to MKL installer --- .install/install_mkl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.install/install_mkl.sh b/.install/install_mkl.sh index 617ef36a7..522896b01 100644 --- a/.install/install_mkl.sh +++ b/.install/install_mkl.sh @@ -12,7 +12,7 @@ then then wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/${uuid}/${filename} chmod u+x ./${filename} - $MODE ./${filename} -a --action install --silent --eula accept --components intel.oneapi.lin.mkl.devel --install-dir ${prefix} + $MODE ./${filename} -a --action install --silent --ignore-errors --eula accept --components intel.oneapi.lin.mkl.devel --install-dir ${prefix} #source ${prefix}/setvars.sh fi fi From 49c129ec7806338c96bf0456f26ce2be712552de Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Tue, 8 Apr 2025 16:33:48 +0200 Subject: [PATCH 05/32] Add GCC version check for SVS --- src/VecSim/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/VecSim/CMakeLists.txt b/src/VecSim/CMakeLists.txt index cd2b9b202..fc76f7ef6 100644 --- a/src/VecSim/CMakeLists.txt +++ b/src/VecSim/CMakeLists.txt @@ -29,6 +29,15 @@ else() message(STATUS "SVS is not supported on this architecture") endif() +# GCC < v11 does not support C++20 features required for SVS +# https://gcc.gnu.org/projects/cxx-status.html +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11.0") + set(SVS_SUPPORTED 0) + message(STATUS "Insufficient gcc version for SVS") + endif() +endif() + # USE_SVS option forsibly OFF if CPU is not supported # elsewhere let user disable SVS cmake_dependent_option(USE_SVS "Build with SVS library support" ON "SVS_SUPPORTED" OFF) From f31ae1714e1922a7d91ae5bd9bf86c36ae03cdd1 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Tue, 8 Apr 2025 16:55:23 +0200 Subject: [PATCH 06/32] Fix Alpine3 compilation error --- .install/alpine_linux_3.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.install/alpine_linux_3.sh b/.install/alpine_linux_3.sh index 4766d44ee..5fc4bed04 100644 --- a/.install/alpine_linux_3.sh +++ b/.install/alpine_linux_3.sh @@ -4,6 +4,6 @@ set -e $MODE apk update -$MODE apk add --no-cache build-base gcc g++ make wget git valgrind +$MODE apk add --no-cache build-base gcc g++ make wget git valgrind linux-headers $MODE apk add --no-cache cmake From 2e7a1691bc716c959a141580821b22a7b0df3a8e Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Tue, 8 Apr 2025 17:40:09 +0200 Subject: [PATCH 07/32] Enable SVS_SHARED_LIB ON --- src/VecSim/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/VecSim/CMakeLists.txt b/src/VecSim/CMakeLists.txt index fc76f7ef6..a6e41d7a1 100644 --- a/src/VecSim/CMakeLists.txt +++ b/src/VecSim/CMakeLists.txt @@ -48,7 +48,7 @@ else() endif() # Configure SVS build -set(SVS_SHARED_LIB OFF CACHE BOOL "Use SVS pre-compiled shared library") +set(SVS_SHARED_LIB ON CACHE BOOL "Use SVS pre-compiled shared library") set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.7/svs-shared-library-0.0.7-avx2.tar.gz" CACHE STRING "SVS URL") set(MKL_HINTS From 7cd754bf7a052b783469df45145cca90252dae64 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Tue, 8 Apr 2025 19:10:58 +0200 Subject: [PATCH 08/32] Install MKL on Ubuntu/Debian via APT --- .install/debian_gnu_linux_11.sh | 2 +- .install/install_mkl_apt.sh | 16 ++++++++++++++++ .install/ubuntu_16.04.sh | 2 +- .install/ubuntu_18.04.sh | 2 +- .install/ubuntu_20.04.sh | 2 +- .install/ubuntu_22.04.sh | 2 +- .install/ubuntu_24.04.sh | 2 +- 7 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 .install/install_mkl_apt.sh diff --git a/.install/debian_gnu_linux_11.sh b/.install/debian_gnu_linux_11.sh index f0d741995..97838cd92 100644 --- a/.install/debian_gnu_linux_11.sh +++ b/.install/debian_gnu_linux_11.sh @@ -6,4 +6,4 @@ MODE=$1 # whether to install using sudo or not $MODE apt update -qq $MODE apt install -yqq git wget build-essential lcov valgrind source install_cmake.sh $MODE -source install_mkl.sh $MODE +source install_mkl_apt.sh $MODE diff --git a/.install/install_mkl_apt.sh b/.install/install_mkl_apt.sh new file mode 100644 index 000000000..12e30072a --- /dev/null +++ b/.install/install_mkl_apt.sh @@ -0,0 +1,16 @@ +#!/bin/bash +version=2024.1 +processor=$(uname -m) +OS_TYPE=$(uname -s) +MODE=$1 # whether to install using sudo or not + +if [[ $OS_TYPE = 'Linux' ]] +then + if [[ $processor = 'x86_64' ]] + then + wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | $MODE tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null + echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | $MODE tee /etc/apt/sources.list.d/intel-oneapi.list + $MODE apt update -qq || true + $MODE apt install -yqq intel-oneapi-mkl-devel-${version} + fi +fi diff --git a/.install/ubuntu_16.04.sh b/.install/ubuntu_16.04.sh index 9cec6ee52..5368e6dc1 100755 --- a/.install/ubuntu_16.04.sh +++ b/.install/ubuntu_16.04.sh @@ -12,4 +12,4 @@ $MODE apt update $MODE apt-get install -yqq git wget make valgrind gcc-9 g++-9 $MODE update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-9 source install_cmake.sh $MODE -source install_mkl.sh $MODE +source install_mkl_apt.sh $MODE diff --git a/.install/ubuntu_18.04.sh b/.install/ubuntu_18.04.sh index d1c812342..670875ea2 100644 --- a/.install/ubuntu_18.04.sh +++ b/.install/ubuntu_18.04.sh @@ -21,4 +21,4 @@ $MODE make install cd .. source install_cmake.sh $MODE -source install_mkl.sh $MODE +source install_mkl_apt.sh $MODE diff --git a/.install/ubuntu_20.04.sh b/.install/ubuntu_20.04.sh index 08282ccac..522893f26 100755 --- a/.install/ubuntu_20.04.sh +++ b/.install/ubuntu_20.04.sh @@ -12,4 +12,4 @@ $MODE update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 60 --slave # align gcov version with gcc version $MODE update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-11 60 source install_cmake.sh $MODE -source install_mkl.sh $MODE +source install_mkl_apt.sh $MODE diff --git a/.install/ubuntu_22.04.sh b/.install/ubuntu_22.04.sh index 1a91daf12..8263bd60f 100755 --- a/.install/ubuntu_22.04.sh +++ b/.install/ubuntu_22.04.sh @@ -9,4 +9,4 @@ $MODE update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 60 --slave # align gcov version with gcc version $MODE update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-12 60 source install_cmake.sh $MODE -source install_mkl.sh $MODE +source install_mkl_apt.sh $MODE diff --git a/.install/ubuntu_24.04.sh b/.install/ubuntu_24.04.sh index ee96eb00a..13234ea9b 100755 --- a/.install/ubuntu_24.04.sh +++ b/.install/ubuntu_24.04.sh @@ -7,4 +7,4 @@ $MODE apt update -qq $MODE apt install -yqq git wget build-essential lcov openssl libssl-dev \ python3 python3-venv python3-dev unzip rsync clang curl source install_cmake.sh $MODE -source install_mkl.sh $MODE +source install_mkl_apt.sh $MODE From 0d40e5f7a586915cfa69a16fe66eb202634500bc Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Wed, 9 Apr 2025 10:46:00 +0200 Subject: [PATCH 09/32] Reduce MKL size in Ubuntu by using 'core' package. --- .install/install_mkl_apt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.install/install_mkl_apt.sh b/.install/install_mkl_apt.sh index 12e30072a..135524d55 100644 --- a/.install/install_mkl_apt.sh +++ b/.install/install_mkl_apt.sh @@ -11,6 +11,6 @@ then wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | $MODE tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | $MODE tee /etc/apt/sources.list.d/intel-oneapi.list $MODE apt update -qq || true - $MODE apt install -yqq intel-oneapi-mkl-devel-${version} + $MODE apt install -yqq intel-oneapi-mkl-core-devel-${version} fi fi From 1a10a0a7d4e1c2d0ac3a14236ae40fb0f25bb15b Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Wed, 9 Apr 2025 11:43:16 +0200 Subject: [PATCH 10/32] durtyfix svs binary package issue --- src/VecSim/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/VecSim/CMakeLists.txt b/src/VecSim/CMakeLists.txt index a6e41d7a1..67d793d5d 100644 --- a/src/VecSim/CMakeLists.txt +++ b/src/VecSim/CMakeLists.txt @@ -71,6 +71,7 @@ if(USE_SVS) FetchContent_Declare( svs URL "${SVS_URL}" + PATCH_COMMAND ${CMAKE_COMMAND} -E create_symlink lib lib64 # durtyfix svs binary package issue ) FetchContent_MakeAvailable(svs) list(APPEND CMAKE_PREFIX_PATH "${svs_SOURCE_DIR}") From e815ade5e58de93689f792e69c55a69e96be6444 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Thu, 10 Apr 2025 17:13:06 +0200 Subject: [PATCH 11/32] Disable SVS share lib if MKL not found --- src/VecSim/CMakeLists.txt | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/VecSim/CMakeLists.txt b/src/VecSim/CMakeLists.txt index 67d793d5d..4d834cc4d 100644 --- a/src/VecSim/CMakeLists.txt +++ b/src/VecSim/CMakeLists.txt @@ -38,7 +38,7 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") endif() endif() -# USE_SVS option forsibly OFF if CPU is not supported +# USE_SVS option forsibly OFF if CPU or compiler is not supported # elsewhere let user disable SVS cmake_dependent_option(USE_SVS "Build with SVS library support" ON "SVS_SUPPORTED" OFF) if(USE_SVS) @@ -51,17 +51,21 @@ endif() set(SVS_SHARED_LIB ON CACHE BOOL "Use SVS pre-compiled shared library") set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.7/svs-shared-library-0.0.7-avx2.tar.gz" CACHE STRING "SVS URL") -set(MKL_HINTS - ${MKLROOT} - $ENV{MKL_DIR} - $ENV{MKL_ROOT} - $ENV{MKLROOT}) +if(SVS_SHARED_LIB) + # try to find MKL + set(MKL_HINTS ${MKLROOT} $ENV{MKL_DIR} $ENV{MKL_ROOT} $ENV{MKLROOT}) + if(LINUX) + list(APPEND MKL_HINTS "/opt/intel/oneapi/mkl/latest") + endif() + if(WIN32) + list(APPEND MKL_HINTS "C:/Program Files (x86)/Intel/oneAPI/mkl/latest") + endif() -if(LINUX) - list(APPEND MKL_HINTS "/opt/intel/oneapi/mkl/latest") -endif() -if(WIN32) - list(APPEND MKL_HINTS "C:/Program Files (x86)/Intel/oneAPI/mkl/latest") + find_package(MKL HINTS ${MKL_HINTS}) + if(NOT MKL_FOUND) + message(WARNING "MKL not found - disabling SVS shared library") + set(SVS_SHARED_LIB OFF FORCE) + endif() endif() if(USE_SVS) @@ -75,7 +79,7 @@ if(USE_SVS) ) FetchContent_MakeAvailable(svs) list(APPEND CMAKE_PREFIX_PATH "${svs_SOURCE_DIR}") - find_package(MKL REQUIRED HINTS ${MKL_HINTS}) + #find_package(MKL REQUIRED HINTS ${MKL_HINTS}) find_package(svs REQUIRED) set(SVS_LVQ_HEADER "svs/extensions/vamana/lvq.h") else() From a037c925b9dfa649c4f142c2c3553628f8ea844b Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Thu, 10 Apr 2025 17:20:03 +0200 Subject: [PATCH 12/32] Revert "durtyfix svs binary package issue" This reverts commit 1a10a0a7d4e1c2d0ac3a14236ae40fb0f25bb15b. --- src/VecSim/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/VecSim/CMakeLists.txt b/src/VecSim/CMakeLists.txt index 4d834cc4d..7650c8109 100644 --- a/src/VecSim/CMakeLists.txt +++ b/src/VecSim/CMakeLists.txt @@ -75,7 +75,6 @@ if(USE_SVS) FetchContent_Declare( svs URL "${SVS_URL}" - PATCH_COMMAND ${CMAKE_COMMAND} -E create_symlink lib lib64 # durtyfix svs binary package issue ) FetchContent_MakeAvailable(svs) list(APPEND CMAKE_PREFIX_PATH "${svs_SOURCE_DIR}") From c39daee6309631c5a384ec047b2f4739bf2cf85c Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Thu, 10 Apr 2025 17:25:10 +0200 Subject: [PATCH 13/32] fixup! Disable SVS share lib if MKL not found --- src/VecSim/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/VecSim/CMakeLists.txt b/src/VecSim/CMakeLists.txt index 7650c8109..1ec9b63d7 100644 --- a/src/VecSim/CMakeLists.txt +++ b/src/VecSim/CMakeLists.txt @@ -64,7 +64,7 @@ if(SVS_SHARED_LIB) find_package(MKL HINTS ${MKL_HINTS}) if(NOT MKL_FOUND) message(WARNING "MKL not found - disabling SVS shared library") - set(SVS_SHARED_LIB OFF FORCE) + set(SVS_SHARED_LIB OFF) endif() endif() From 4a33ae83bed355089b5056092756e7368b5322f9 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Thu, 10 Apr 2025 17:41:31 +0200 Subject: [PATCH 14/32] fixup! Refactor Info Report - [MOD-9321] (#650) --- src/VecSim/algorithms/svs/svs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 6210f6e83..4de31fb8c 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -258,7 +258,7 @@ class SVSIndex : public VecSimIndexAbstract, fl return info; } - VecSimIndexDInfo debugInfo() const override { + VecSimIndexDebugInfo debugInfo() const override { VecSimIndexDebugInfo info; info.commonInfo = this->getCommonInfo(); info.commonInfo.basicInfo.algo = VecSimAlgo_SVS; From 92eae96665998b1fd9da906f0733305d58175a4d Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Thu, 10 Apr 2025 18:29:47 +0200 Subject: [PATCH 15/32] Revert SVS disabling changes made in f6702147 --- Makefile | 2 +- src/VecSim/CMakeLists.txt | 3 +-- tests/unit/CMakeLists.txt | 9 ++++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 832a03849..450194de8 100644 --- a/Makefile +++ b/Makefile @@ -197,7 +197,7 @@ endif flow_test: $(SHOW)poetry install $(POETRY_ARGS) - $(SHOW)poetry run pytest tests/flow/$(TEST) --deselect=tests/flow/test_svs.py -v -s + $(SHOW)poetry run pytest tests/flow/$(TEST) -v -s .PHONY: flow_test diff --git a/src/VecSim/CMakeLists.txt b/src/VecSim/CMakeLists.txt index 1ec9b63d7..d6d2349fc 100644 --- a/src/VecSim/CMakeLists.txt +++ b/src/VecSim/CMakeLists.txt @@ -40,7 +40,7 @@ endif() # USE_SVS option forsibly OFF if CPU or compiler is not supported # elsewhere let user disable SVS -cmake_dependent_option(USE_SVS "Build with SVS library support" ON "SVS_SUPPORTED" OFF) +set(USE_SVS ${SVS_SUPPORTED} CACHE BOOL "Build with SVS library support" PARENT_SCOPE) if(USE_SVS) message(STATUS "SVS support enabled") else() @@ -78,7 +78,6 @@ if(USE_SVS) ) FetchContent_MakeAvailable(svs) list(APPEND CMAKE_PREFIX_PATH "${svs_SOURCE_DIR}") - #find_package(MKL REQUIRED HINTS ${MKL_HINTS}) find_package(svs REQUIRED) set(SVS_LVQ_HEADER "svs/extensions/vamana/lvq.h") else() diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 67a281868..7b1171faf 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -60,6 +60,11 @@ target_link_libraries(test_fp16 PUBLIC gtest_main VectorSimilarity) target_link_libraries(test_int8 PUBLIC gtest_main VectorSimilarity) target_link_libraries(test_uint8 PUBLIC gtest_main VectorSimilarity) +if(USE_SVS) + add_executable(test_svs ../utils/mock_thread_pool.cpp test_svs.cpp unit_test_utils.cpp) + target_link_libraries(test_svs PUBLIC gtest_main VectorSimilarity) +endif() + include(GoogleTest) gtest_discover_tests(test_hnsw) @@ -76,7 +81,5 @@ gtest_discover_tests(test_int8 TEST_PREFIX INT8UNIT_) gtest_discover_tests(test_uint8 TEST_PREFIX UINT8UNIT_) if(USE_SVS) - add_executable(test_svs ../utils/mock_thread_pool.cpp test_svs.cpp unit_test_utils.cpp) - target_link_libraries(test_svs PUBLIC gtest_main VectorSimilarity) gtest_discover_tests(test_svs) -endif() \ No newline at end of file +endif() From d1f3a728b106e4480b1b0b45dcf5f90d4a68ace9 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Fri, 11 Apr 2025 11:46:42 +0200 Subject: [PATCH 16/32] Move SVS cmake checks and definitions to a separate file. --- CMakeLists.txt | 2 + cmake/svs.cmake | 92 +++++++++++++++++++++++++++++ src/VecSim/CMakeLists.txt | 93 +++--------------------------- src/python_bindings/CMakeLists.txt | 1 + tests/unit/test_svs.cpp | 7 --- 5 files changed, 102 insertions(+), 93 deletions(-) create mode 100644 cmake/svs.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index c52d42b84..e3e36de25 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,8 @@ IF(USE_PROFILE) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") ENDIF() +include(cmake/svs.cmake) + include_directories(src) if(VECSIM_BUILD_TESTS) diff --git a/cmake/svs.cmake b/cmake/svs.cmake new file mode 100644 index 000000000..c867caaa1 --- /dev/null +++ b/cmake/svs.cmake @@ -0,0 +1,92 @@ +cmake_minimum_required(VERSION 3.12) +cmake_policy(SET CMP0074 NEW) +if(POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) +endif() + +# option(SVS_EXPERIMENTAL_LEANVEC "Enable experimental LeanVec in SVS" YES) +# if(SVS_EXPERIMENTAL_LEANVEC) +# add_definitions(-DSVS_EXPERIMENTAL_LEANVEC) +# endif() + +if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "(x86_64)|(AMD64|amd64)") + set(SVS_SUPPORTED 1) +else() + set(SVS_SUPPORTED 0) + message(STATUS "SVS is not supported on this architecture") +endif() + +# GCC < v11 does not support C++20 features required for SVS +# https://gcc.gnu.org/projects/cxx-status.html +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11.0") + set(SVS_SUPPORTED 0) + message(STATUS "Insufficient gcc version for SVS") + endif() +endif() + +include(CMakeDependentOption) + +# USE_SVS option forsibly OFF if CPU or compiler is not supported +# elsewhere let user disable SVS +cmake_dependent_option(USE_SVS "Build with SVS library support" ON "SVS_SUPPORTED" OFF) +if(USE_SVS) + message(STATUS "SVS support enabled") +else() + message(STATUS "SVS support disabled") +endif() + +# try to find MKL +set(MKL_HINTS ${MKLROOT} $ENV{MKL_DIR} $ENV{MKL_ROOT} $ENV{MKLROOT}) +if(LINUX) + list(APPEND MKL_HINTS "/opt/intel/oneapi/mkl/latest") +endif() +if(WIN32) + list(APPEND MKL_HINTS "C:/Program Files (x86)/Intel/oneAPI/mkl/latest") +endif() +find_package(MKL HINTS ${MKL_HINTS}) +if(NOT MKL_FOUND) + message(WARNING "MKL not found - disabling SVS shared library") +endif() + +# Configure SVS build +cmake_dependent_option(SVS_SHARED_LIB "Use SVS pre-compiled shared library" ON "SVS_SUPPORTED AND MKL_FOUND" OFF) +set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.7/svs-shared-library-0.0.7-avx2.tar.gz" CACHE STRING "SVS URL") + +if(USE_SVS) + add_compile_definitions("HAVE_SVS=1") + set(svs_factory_file "index_factories/svs_factory.cpp") + set(SVS_TARGET_NAME "svs::svs") + if(SVS_SHARED_LIB) + include(FetchContent) + FetchContent_Declare( + svs + URL "${SVS_URL}" + ) + FetchContent_MakeAvailable(svs) + list(APPEND CMAKE_PREFIX_PATH "${svs_SOURCE_DIR}") + find_package(svs REQUIRED) + set(SVS_LVQ_HEADER "svs/extensions/vamana/lvq.h") + set(SVS_TARGET_NAME "svs::svs_shared_library") + else() + # this file is included from python_bindings/CMakeLists.txt + # SVS sources path to be relative to this file + get_filename_component(root ${CMAKE_CURRENT_LIST_DIR}/.. ABSOLUTE) + add_subdirectory( + ${root}/deps/ScalableVectorSearch + deps/ScalableVectorSearch + ) + set(SVS_LVQ_HEADER "svs/quantization/lvq/impl/lvq_impl.h") + set(SVS_TARGET_NAME "svs::svs") + endif() +else() + add_compile_definitions("HAVE_SVS=0") +endif() + +if(EXISTS "${svs_SOURCE_DIR}/include/${SVS_LVQ_HEADER}") + message("SVS LVQ implementation found") + add_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=1" PUBLIC "SVS_LVQ_HEADER=\"${SVS_LVQ_HEADER}\"") +else() + message("SVS LVQ implementation not found") + add_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=0") +endif() diff --git a/src/VecSim/CMakeLists.txt b/src/VecSim/CMakeLists.txt index d6d2349fc..cb23e46fe 100644 --- a/src/VecSim/CMakeLists.txt +++ b/src/VecSim/CMakeLists.txt @@ -17,83 +17,14 @@ set(CMAKE_CXX_FLAGS_DEBUG "-g") set(CMAKE_CXX_FLAGS_RELEASE "-O3") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g") -# option(SVS_EXPERIMENTAL_LEANVEC "Enable experimental LeanVec in SVS" YES) -# if(SVS_EXPERIMENTAL_LEANVEC) -# add_definitions(-DSVS_EXPERIMENTAL_LEANVEC) -# endif() - -if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "(x86_64)|(AMD64|amd64)") - set(SVS_SUPPORTED 1) -else() - set(SVS_SUPPORTED 0) - message(STATUS "SVS is not supported on this architecture") -endif() - -# GCC < v11 does not support C++20 features required for SVS -# https://gcc.gnu.org/projects/cxx-status.html -if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11.0") - set(SVS_SUPPORTED 0) - message(STATUS "Insufficient gcc version for SVS") - endif() -endif() - -# USE_SVS option forsibly OFF if CPU or compiler is not supported -# elsewhere let user disable SVS -set(USE_SVS ${SVS_SUPPORTED} CACHE BOOL "Build with SVS library support" PARENT_SCOPE) -if(USE_SVS) - message(STATUS "SVS support enabled") -else() - message(STATUS "SVS support disabled") -endif() - -# Configure SVS build -set(SVS_SHARED_LIB ON CACHE BOOL "Use SVS pre-compiled shared library") -set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.7/svs-shared-library-0.0.7-avx2.tar.gz" CACHE STRING "SVS URL") - -if(SVS_SHARED_LIB) - # try to find MKL - set(MKL_HINTS ${MKLROOT} $ENV{MKL_DIR} $ENV{MKL_ROOT} $ENV{MKLROOT}) - if(LINUX) - list(APPEND MKL_HINTS "/opt/intel/oneapi/mkl/latest") - endif() - if(WIN32) - list(APPEND MKL_HINTS "C:/Program Files (x86)/Intel/oneAPI/mkl/latest") - endif() +add_subdirectory(spaces) - find_package(MKL HINTS ${MKL_HINTS}) - if(NOT MKL_FOUND) - message(WARNING "MKL not found - disabling SVS shared library") - set(SVS_SHARED_LIB OFF) - endif() -endif() +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall") if(USE_SVS) set(svs_factory_file "index_factories/svs_factory.cpp") - if(SVS_SHARED_LIB) - include(FetchContent) - FetchContent_Declare( - svs - URL "${SVS_URL}" - ) - FetchContent_MakeAvailable(svs) - list(APPEND CMAKE_PREFIX_PATH "${svs_SOURCE_DIR}") - find_package(svs REQUIRED) - set(SVS_LVQ_HEADER "svs/extensions/vamana/lvq.h") - else() - get_filename_component(root ${CMAKE_CURRENT_LIST_DIR}/../.. ABSOLUTE) - add_subdirectory( - ${root}/deps/ScalableVectorSearch - deps/ScalableVectorSearch - ) - set(SVS_LVQ_HEADER "svs/quantization/lvq/impl/lvq_impl.h") - endif() endif() -add_subdirectory(spaces) - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall") - add_library(VectorSimilarity ${VECSIM_LIBTYPE} index_factories/brute_force_factory.cpp index_factories/hnsw_factory.cpp @@ -115,23 +46,13 @@ add_library(VectorSimilarity ${VECSIM_LIBTYPE} ${HEADER_LIST} ) -if (USE_SVS) - target_link_libraries(VectorSimilarity VectorSimilaritySpaces svs::svs) +target_link_libraries(VectorSimilarity VectorSimilaritySpaces) + +if (TARGET svs::svs) + target_link_libraries(VectorSimilarity svs::svs) if(TARGET svs::svs_shared_library) - target_link_libraries(VectorSimilarity svs::svs_shared_library MKL::MKL) + target_link_libraries(VectorSimilarity svs::svs_shared_library) endif() - target_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS=1") -else() - target_link_libraries(VectorSimilarity VectorSimilaritySpaces) - target_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS=0") -endif() - -if(EXISTS "${svs_SOURCE_DIR}/include/${SVS_LVQ_HEADER}") - message("SVS LVQ implementation found") - target_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=1" PUBLIC "SVS_LVQ_HEADER=\"${SVS_LVQ_HEADER}\"") -else() - message("SVS LVQ implementation not found") - target_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=0") endif() if(VECSIM_BUILD_TESTS) diff --git a/src/python_bindings/CMakeLists.txt b/src/python_bindings/CMakeLists.txt index de51f0a27..48d4a71fa 100644 --- a/src/python_bindings/CMakeLists.txt +++ b/src/python_bindings/CMakeLists.txt @@ -20,6 +20,7 @@ if(NOT pybind11_POPULATED) add_subdirectory(${pybind11_SOURCE_DIR} ${pybind11_BINARY_DIR}) endif() +include(${root}/cmake/svs.cmake) add_subdirectory(${root}/src/VecSim VectorSimilarity) include_directories(${root}/src ${root}/tests/utils) diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index 1f12ec0a6..2234aee1c 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -1178,13 +1178,6 @@ TYPED_TEST(SVSTest, svs_test_inf_score) { .dim = dim, .metric = VecSimMetric_L2, .blockSize = 1, - /* SVS-Vamana specifics */ - .alpha = 1.2, - .graph_max_degree = 64, - .construction_window_size = 20, - .max_candidate_pool_size = 1024, - .prune_to = 60, - .use_search_history = VecSimOption_ENABLE, }; VecSimIndex *index = this->CreateNewIndex(params); From f56a729fe5167320d299505be24f34321db4aad4 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Fri, 11 Apr 2025 17:23:59 +0200 Subject: [PATCH 17/32] Disable SVS AVX512 for Valgrind Debug build --- cmake/svs.cmake | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index c867caaa1..153b41bfc 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -9,6 +9,13 @@ endif() # add_definitions(-DSVS_EXPERIMENTAL_LEANVEC) # endif() +# Valgrind does not support AVX512 and Valgrind in running in Debug +# so disable it if we are in Debug mode +string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) +if(uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG") + set(SVS_NO_AVX512 ON) +endif() + if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "(x86_64)|(AMD64|amd64)") set(SVS_SUPPORTED 1) else() From d4560946b49903c1a0f599bb49032e33ec923cc6 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Mon, 14 Apr 2025 14:57:44 +0200 Subject: [PATCH 18/32] Fix Valgrind tests --- cmake/svs.cmake | 18 ++++++++++++++++++ cmake/svs_allocator.patch | 13 +++++++++++++ tests/unit/test_svs.cpp | 2 +- 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 cmake/svs_allocator.patch diff --git a/cmake/svs.cmake b/cmake/svs.cmake index 153b41bfc..e296fef37 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -75,6 +75,7 @@ if(USE_SVS) find_package(svs REQUIRED) set(SVS_LVQ_HEADER "svs/extensions/vamana/lvq.h") set(SVS_TARGET_NAME "svs::svs_shared_library") + set(_svs_patch_dir "${svs_SOURCE_DIR}") else() # this file is included from python_bindings/CMakeLists.txt # SVS sources path to be relative to this file @@ -85,7 +86,24 @@ if(USE_SVS) ) set(SVS_LVQ_HEADER "svs/quantization/lvq/impl/lvq_impl.h") set(SVS_TARGET_NAME "svs::svs") + + # begin durtyfix svs allocator issue + find_path(_svs_patch_dir + NAMES "include/svs/lib/memory.h" + PATHS ${root}/deps/ScalableVectorSearch + PATH_SUFFIXES "ScalableVectorSearch" + REQUIRED NO_DEFAULT_PATH NO_CMAKE_FIND_ROOT_PATH + NO_CACHE + ) + # end durtifix endif() + # begin durtyfix svs allocator issue + find_package(Patch) + set(_svs_alloc_patch_args "-f" "-p" "1" "-i" "${CMAKE_CURRENT_LIST_DIR}/svs_allocator.patch" ) + execute_process(COMMAND ${Patch_EXECUTABLE} ${_svs_alloc_patch_args} + WORKING_DIRECTORY ${_svs_patch_dir} + ) + # end durtifix else() add_compile_definitions("HAVE_SVS=0") endif() diff --git a/cmake/svs_allocator.patch b/cmake/svs_allocator.patch new file mode 100644 index 000000000..b34475cd7 --- /dev/null +++ b/cmake/svs_allocator.patch @@ -0,0 +1,13 @@ +diff --git a/include/svs/lib/memory.h b/include/svs/lib/memory.h +index c79fff1..dcc015b 100644 +--- a/include/svs/lib/memory.h ++++ b/include/svs/lib/memory.h +@@ -61,7 +61,7 @@ template struct Allocator { + } + + constexpr void deallocate(value_type* ptr, size_t count) noexcept { +- ::operator delete(ptr, count); ++ ::operator delete(ptr, count, std::align_val_t(alignof(T))); + } + + // Intercept zero-argument construction to do default initialization. diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index 2234aee1c..1706040ff 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -484,7 +484,7 @@ TYPED_TEST(SVSTest, svs_batch_iterator_non_unique_scores) { // as the number of results is 5, which is more than 0.1% of the index size. for index of size // 10000, we will run the heap-based search until we return 5000 results, and then switch to // select-based search. - for (size_t n : {100, 10000}) { + for (size_t n : {100, 1000}) { SVSParams params = { .dim = dim, .metric = VecSimMetric_L2, From 7a11209cb2c21d505e19c354b51174a02fabbd81 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Mon, 14 Apr 2025 16:38:30 +0200 Subject: [PATCH 19/32] Disable SVS shared library for Valgrind tests --- .github/workflows/task-unit-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/task-unit-test.yml b/.github/workflows/task-unit-test.yml index f18b2195e..edff79b90 100644 --- a/.github/workflows/task-unit-test.yml +++ b/.github/workflows/task-unit-test.yml @@ -49,7 +49,7 @@ jobs: run: make unit_test - name: valgrind if: ${{ inputs.run-valgrind }} - run: make valgrind + run: make valgrind CMAKE_FLAGS="-DSVS_SHARED_LIB=OFF" - name: Archive valgrind tests reports if: ${{ inputs.run-valgrind && failure() }} uses: actions/upload-artifact@v4 From 82233854471e5fabdc859607c156faa77c957120 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Mon, 14 Apr 2025 18:56:20 +0200 Subject: [PATCH 20/32] Update svs_allocator.patch --- cmake/svs_allocator.patch | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/cmake/svs_allocator.patch b/cmake/svs_allocator.patch index b34475cd7..1094ea023 100644 --- a/cmake/svs_allocator.patch +++ b/cmake/svs_allocator.patch @@ -1,5 +1,5 @@ diff --git a/include/svs/lib/memory.h b/include/svs/lib/memory.h -index c79fff1..dcc015b 100644 +index c79fff1..7f55390 100644 --- a/include/svs/lib/memory.h +++ b/include/svs/lib/memory.h @@ -61,7 +61,7 @@ template struct Allocator { @@ -7,7 +7,21 @@ index c79fff1..dcc015b 100644 constexpr void deallocate(value_type* ptr, size_t count) noexcept { - ::operator delete(ptr, count); -+ ::operator delete(ptr, count, std::align_val_t(alignof(T))); ++ ::operator delete(static_cast(ptr), sizeof(T) * count, std::align_val_t(alignof(T))); } // Intercept zero-argument construction to do default initialization. +diff --git a/include/svs/lib/threads/threadlocal.h b/include/svs/lib/threads/threadlocal.h +index f6d399f..5fdebd2 100644 +--- a/include/svs/lib/threads/threadlocal.h ++++ b/include/svs/lib/threads/threadlocal.h +@@ -82,7 +82,8 @@ template struct AlignedAllocat + } + + void deallocate(value_type* ptr, size_t count) noexcept { +- ::operator delete(ptr, count, std::align_val_t{alignment}); ++ size_t bytes = alignment * lib::div_round_up(sizeof(T) * count, alignment); ++ ::operator delete(static_cast(ptr), bytes, std::align_val_t{alignment}); + } + }; + From 7b9c5ad8237fa31f8f02e6d5c81f7a3553f56738 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Tue, 15 Apr 2025 11:32:39 +0200 Subject: [PATCH 21/32] Disable SVS shared library by default --- cmake/svs.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index e296fef37..0dc2ab799 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -57,7 +57,7 @@ if(NOT MKL_FOUND) endif() # Configure SVS build -cmake_dependent_option(SVS_SHARED_LIB "Use SVS pre-compiled shared library" ON "SVS_SUPPORTED AND MKL_FOUND" OFF) +cmake_dependent_option(SVS_SHARED_LIB "Use SVS pre-compiled shared library" OFF "SVS_SUPPORTED AND MKL_FOUND" OFF) set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.7/svs-shared-library-0.0.7-avx2.tar.gz" CACHE STRING "SVS URL") if(USE_SVS) From 6673c3b9720a9ef7785486a57f07bc2dc72b69fe Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Tue, 15 Apr 2025 14:23:43 +0200 Subject: [PATCH 22/32] Add 'patch' tool installation required for SVS to rocky9 --- .install/rocky_linux_9.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.install/rocky_linux_9.sh b/.install/rocky_linux_9.sh index 05f83fef5..c2d7acfe1 100644 --- a/.install/rocky_linux_9.sh +++ b/.install/rocky_linux_9.sh @@ -5,6 +5,9 @@ export DEBIAN_FRONTEND=noninteractive $MODE dnf update -y $MODE dnf install -y gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ make wget git valgrind +# Install patch tool for SVS +$MODE dnf install -y patch + cp /opt/rh/gcc-toolset-13/enable /etc/profile.d/gcc-toolset-13.sh source install_cmake.sh $MODE From b4cdc4bd3a9120400aabf1173af63d7ec2dad641 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Tue, 15 Apr 2025 17:20:20 +0200 Subject: [PATCH 23/32] Revert "Disable SVS shared library by default" This reverts commit 7b9c5ad8237fa31f8f02e6d5c81f7a3553f56738. --- cmake/svs.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index 0dc2ab799..e296fef37 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -57,7 +57,7 @@ if(NOT MKL_FOUND) endif() # Configure SVS build -cmake_dependent_option(SVS_SHARED_LIB "Use SVS pre-compiled shared library" OFF "SVS_SUPPORTED AND MKL_FOUND" OFF) +cmake_dependent_option(SVS_SHARED_LIB "Use SVS pre-compiled shared library" ON "SVS_SUPPORTED AND MKL_FOUND" OFF) set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.7/svs-shared-library-0.0.7-avx2.tar.gz" CACHE STRING "SVS URL") if(USE_SVS) From ee12dbef56a95663f497ae08cc2590dbb1d0a2c8 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Wed, 16 Apr 2025 11:36:20 +0200 Subject: [PATCH 24/32] Update SVS submodule and remove allocator patch --- cmake/svs.cmake | 18 ------------------ cmake/svs_allocator.patch | 27 --------------------------- deps/ScalableVectorSearch | 2 +- 3 files changed, 1 insertion(+), 46 deletions(-) delete mode 100644 cmake/svs_allocator.patch diff --git a/cmake/svs.cmake b/cmake/svs.cmake index e296fef37..153b41bfc 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -75,7 +75,6 @@ if(USE_SVS) find_package(svs REQUIRED) set(SVS_LVQ_HEADER "svs/extensions/vamana/lvq.h") set(SVS_TARGET_NAME "svs::svs_shared_library") - set(_svs_patch_dir "${svs_SOURCE_DIR}") else() # this file is included from python_bindings/CMakeLists.txt # SVS sources path to be relative to this file @@ -86,24 +85,7 @@ if(USE_SVS) ) set(SVS_LVQ_HEADER "svs/quantization/lvq/impl/lvq_impl.h") set(SVS_TARGET_NAME "svs::svs") - - # begin durtyfix svs allocator issue - find_path(_svs_patch_dir - NAMES "include/svs/lib/memory.h" - PATHS ${root}/deps/ScalableVectorSearch - PATH_SUFFIXES "ScalableVectorSearch" - REQUIRED NO_DEFAULT_PATH NO_CMAKE_FIND_ROOT_PATH - NO_CACHE - ) - # end durtifix endif() - # begin durtyfix svs allocator issue - find_package(Patch) - set(_svs_alloc_patch_args "-f" "-p" "1" "-i" "${CMAKE_CURRENT_LIST_DIR}/svs_allocator.patch" ) - execute_process(COMMAND ${Patch_EXECUTABLE} ${_svs_alloc_patch_args} - WORKING_DIRECTORY ${_svs_patch_dir} - ) - # end durtifix else() add_compile_definitions("HAVE_SVS=0") endif() diff --git a/cmake/svs_allocator.patch b/cmake/svs_allocator.patch deleted file mode 100644 index 1094ea023..000000000 --- a/cmake/svs_allocator.patch +++ /dev/null @@ -1,27 +0,0 @@ -diff --git a/include/svs/lib/memory.h b/include/svs/lib/memory.h -index c79fff1..7f55390 100644 ---- a/include/svs/lib/memory.h -+++ b/include/svs/lib/memory.h -@@ -61,7 +61,7 @@ template struct Allocator { - } - - constexpr void deallocate(value_type* ptr, size_t count) noexcept { -- ::operator delete(ptr, count); -+ ::operator delete(static_cast(ptr), sizeof(T) * count, std::align_val_t(alignof(T))); - } - - // Intercept zero-argument construction to do default initialization. -diff --git a/include/svs/lib/threads/threadlocal.h b/include/svs/lib/threads/threadlocal.h -index f6d399f..5fdebd2 100644 ---- a/include/svs/lib/threads/threadlocal.h -+++ b/include/svs/lib/threads/threadlocal.h -@@ -82,7 +82,8 @@ template struct AlignedAllocat - } - - void deallocate(value_type* ptr, size_t count) noexcept { -- ::operator delete(ptr, count, std::align_val_t{alignment}); -+ size_t bytes = alignment * lib::div_round_up(sizeof(T) * count, alignment); -+ ::operator delete(static_cast(ptr), bytes, std::align_val_t{alignment}); - } - }; - diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch index 1e59bf6a9..5a7da989d 160000 --- a/deps/ScalableVectorSearch +++ b/deps/ScalableVectorSearch @@ -1 +1 @@ -Subproject commit 1e59bf6a9db91886dbded8bf18e15fb1ad8e3e44 +Subproject commit 5a7da989d0d0e5a603851069c90465498845a78b From 79c56826a4bd1c2c5d371e3eefd09192b9ba73a6 Mon Sep 17 00:00:00 2001 From: meiravgri Date: Wed, 16 Apr 2025 12:43:34 +0000 Subject: [PATCH 25/32] enable mac tests disable skipping --- tests/unit/CMakeLists.txt | 8 ++++---- tests/unit/test_svs.cpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 7b1171faf..d3d018b49 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -60,10 +60,10 @@ target_link_libraries(test_fp16 PUBLIC gtest_main VectorSimilarity) target_link_libraries(test_int8 PUBLIC gtest_main VectorSimilarity) target_link_libraries(test_uint8 PUBLIC gtest_main VectorSimilarity) -if(USE_SVS) +# if(USE_SVS) add_executable(test_svs ../utils/mock_thread_pool.cpp test_svs.cpp unit_test_utils.cpp) target_link_libraries(test_svs PUBLIC gtest_main VectorSimilarity) -endif() +# endif() include(GoogleTest) @@ -80,6 +80,6 @@ gtest_discover_tests(test_fp16 TEST_PREFIX FP16UNIT_) gtest_discover_tests(test_int8 TEST_PREFIX INT8UNIT_) gtest_discover_tests(test_uint8 TEST_PREFIX UINT8UNIT_) -if(USE_SVS) +# if(USE_SVS) gtest_discover_tests(test_svs) -endif() +# endif() diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index 1706040ff..4f86e031a 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -38,7 +38,7 @@ class SVSTest : public ::testing::Test { void SetUp() override { if constexpr (index_type_t::get_quant_bits() != VecSimSvsQuant_NONE) if (!_checkCPU()) { - GTEST_SKIP() << "SVS LVQ is not supported on non-Intel hardware."; + // GTEST_SKIP() << "SVS LVQ is not supported on non-Intel hardware."; } } }; From d67977a84834d199b0a85deecbaf970b0371eec9 Mon Sep 17 00:00:00 2001 From: meiravgri Date: Wed, 16 Apr 2025 13:10:57 +0000 Subject: [PATCH 26/32] Revert "enable mac tests" This reverts commit 79c56826a4bd1c2c5d371e3eefd09192b9ba73a6. --- tests/unit/CMakeLists.txt | 8 ++++---- tests/unit/test_svs.cpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index d3d018b49..7b1171faf 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -60,10 +60,10 @@ target_link_libraries(test_fp16 PUBLIC gtest_main VectorSimilarity) target_link_libraries(test_int8 PUBLIC gtest_main VectorSimilarity) target_link_libraries(test_uint8 PUBLIC gtest_main VectorSimilarity) -# if(USE_SVS) +if(USE_SVS) add_executable(test_svs ../utils/mock_thread_pool.cpp test_svs.cpp unit_test_utils.cpp) target_link_libraries(test_svs PUBLIC gtest_main VectorSimilarity) -# endif() +endif() include(GoogleTest) @@ -80,6 +80,6 @@ gtest_discover_tests(test_fp16 TEST_PREFIX FP16UNIT_) gtest_discover_tests(test_int8 TEST_PREFIX INT8UNIT_) gtest_discover_tests(test_uint8 TEST_PREFIX UINT8UNIT_) -# if(USE_SVS) +if(USE_SVS) gtest_discover_tests(test_svs) -# endif() +endif() diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index 4f86e031a..1706040ff 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -38,7 +38,7 @@ class SVSTest : public ::testing::Test { void SetUp() override { if constexpr (index_type_t::get_quant_bits() != VecSimSvsQuant_NONE) if (!_checkCPU()) { - // GTEST_SKIP() << "SVS LVQ is not supported on non-Intel hardware."; + GTEST_SKIP() << "SVS LVQ is not supported on non-Intel hardware."; } } }; From 2138ca594ae6e2e62250ececc2e4cc06c6cb6d18 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Wed, 16 Apr 2025 16:38:24 +0200 Subject: [PATCH 27/32] Address code review comments --- .install/rocky_linux_9.sh | 3 --- cmake/svs.cmake | 43 +++++++++++++++++++-------------------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/.install/rocky_linux_9.sh b/.install/rocky_linux_9.sh index c2d7acfe1..05f83fef5 100644 --- a/.install/rocky_linux_9.sh +++ b/.install/rocky_linux_9.sh @@ -5,9 +5,6 @@ export DEBIAN_FRONTEND=noninteractive $MODE dnf update -y $MODE dnf install -y gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ make wget git valgrind -# Install patch tool for SVS -$MODE dnf install -y patch - cp /opt/rh/gcc-toolset-13/enable /etc/profile.d/gcc-toolset-13.sh source install_cmake.sh $MODE diff --git a/cmake/svs.cmake b/cmake/svs.cmake index 153b41bfc..4408850b0 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -4,11 +4,6 @@ if(POLICY CMP0135) cmake_policy(SET CMP0135 NEW) endif() -# option(SVS_EXPERIMENTAL_LEANVEC "Enable experimental LeanVec in SVS" YES) -# if(SVS_EXPERIMENTAL_LEANVEC) -# add_definitions(-DSVS_EXPERIMENTAL_LEANVEC) -# endif() - # Valgrind does not support AVX512 and Valgrind in running in Debug # so disable it if we are in Debug mode string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) @@ -34,30 +29,34 @@ endif() include(CMakeDependentOption) -# USE_SVS option forsibly OFF if CPU or compiler is not supported +# USE_SVS option forcibly OFF if CPU or compiler is not supported # elsewhere let user disable SVS cmake_dependent_option(USE_SVS "Build with SVS library support" ON "SVS_SUPPORTED" OFF) if(USE_SVS) - message(STATUS "SVS support enabled") else() - message(STATUS "SVS support disabled") endif() -# try to find MKL -set(MKL_HINTS ${MKLROOT} $ENV{MKL_DIR} $ENV{MKL_ROOT} $ENV{MKLROOT}) -if(LINUX) - list(APPEND MKL_HINTS "/opt/intel/oneapi/mkl/latest") -endif() -if(WIN32) - list(APPEND MKL_HINTS "C:/Program Files (x86)/Intel/oneAPI/mkl/latest") -endif() -find_package(MKL HINTS ${MKL_HINTS}) -if(NOT MKL_FOUND) - message(WARNING "MKL not found - disabling SVS shared library") +if(USE_SVS) + message(STATUS "SVS support enabled") + # try to find MKL + set(MKL_HINTS ${MKLROOT} $ENV{MKL_DIR} $ENV{MKL_ROOT} $ENV{MKLROOT}) + if(LINUX) + list(APPEND MKL_HINTS "/opt/intel/oneapi/mkl/latest") + endif() + if(WIN32) + list(APPEND MKL_HINTS "C:/Program Files (x86)/Intel/oneAPI/mkl/latest") + endif() + find_package(MKL HINTS ${MKL_HINTS}) + if(NOT MKL_FOUND) + message(WARNING "MKL not found - disabling SVS shared library") + endif() +else() + message(STATUS "SVS support disabled") + set(MKL_FOUND FALSE) endif() # Configure SVS build -cmake_dependent_option(SVS_SHARED_LIB "Use SVS pre-compiled shared library" ON "SVS_SUPPORTED AND MKL_FOUND" OFF) +cmake_dependent_option(SVS_SHARED_LIB "Use SVS pre-compiled shared library" ON "USE_SVS AND MKL_FOUND" OFF) set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.7/svs-shared-library-0.0.7-avx2.tar.gz" CACHE STRING "SVS URL") if(USE_SVS) @@ -76,8 +75,8 @@ if(USE_SVS) set(SVS_LVQ_HEADER "svs/extensions/vamana/lvq.h") set(SVS_TARGET_NAME "svs::svs_shared_library") else() - # this file is included from python_bindings/CMakeLists.txt - # SVS sources path to be relative to this file + # this file is included from both CMakeLists.txt and python_bindings/CMakeLists.txt + # so, set SVS sources path relative to this file get_filename_component(root ${CMAKE_CURRENT_LIST_DIR}/.. ABSOLUTE) add_subdirectory( ${root}/deps/ScalableVectorSearch From b71c5d7b650c8199e8196b26018de063ce8eb9fe Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Thu, 17 Apr 2025 11:52:08 +0200 Subject: [PATCH 28/32] Address code review comments s1e2 --- cmake/svs.cmake | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index 4408850b0..a319ec39a 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -32,9 +32,6 @@ include(CMakeDependentOption) # USE_SVS option forcibly OFF if CPU or compiler is not supported # elsewhere let user disable SVS cmake_dependent_option(USE_SVS "Build with SVS library support" ON "SVS_SUPPORTED" OFF) -if(USE_SVS) -else() -endif() if(USE_SVS) message(STATUS "SVS support enabled") @@ -75,8 +72,8 @@ if(USE_SVS) set(SVS_LVQ_HEADER "svs/extensions/vamana/lvq.h") set(SVS_TARGET_NAME "svs::svs_shared_library") else() - # this file is included from both CMakeLists.txt and python_bindings/CMakeLists.txt - # so, set SVS sources path relative to this file + # This file is included from both CMakeLists.txt and python_bindings/CMakeLists.txt + # Set `root` relative to this file, regardless of where it is included from. get_filename_component(root ${CMAKE_CURRENT_LIST_DIR}/.. ABSOLUTE) add_subdirectory( ${root}/deps/ScalableVectorSearch From cbd5b5de9f480e2de24d5f25967fa611a1bd779a Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Thu, 17 Apr 2025 12:09:05 +0200 Subject: [PATCH 29/32] Move all SVS configuration under one if --- cmake/svs.cmake | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index a319ec39a..e367d29ad 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -35,6 +35,10 @@ cmake_dependent_option(USE_SVS "Build with SVS library support" ON "SVS_SUPPORTE if(USE_SVS) message(STATUS "SVS support enabled") + # Configure SVS build + add_compile_definitions("HAVE_SVS=1") + set(svs_factory_file "index_factories/svs_factory.cpp") + # try to find MKL set(MKL_HINTS ${MKLROOT} $ENV{MKL_DIR} $ENV{MKL_ROOT} $ENV{MKLROOT}) if(LINUX) @@ -47,19 +51,10 @@ if(USE_SVS) if(NOT MKL_FOUND) message(WARNING "MKL not found - disabling SVS shared library") endif() -else() - message(STATUS "SVS support disabled") - set(MKL_FOUND FALSE) -endif() -# Configure SVS build -cmake_dependent_option(SVS_SHARED_LIB "Use SVS pre-compiled shared library" ON "USE_SVS AND MKL_FOUND" OFF) -set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.7/svs-shared-library-0.0.7-avx2.tar.gz" CACHE STRING "SVS URL") + cmake_dependent_option(SVS_SHARED_LIB "Use SVS pre-compiled shared library" ON "USE_SVS AND MKL_FOUND" OFF) + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.7/svs-shared-library-0.0.7-avx2.tar.gz" CACHE STRING "SVS URL") -if(USE_SVS) - add_compile_definitions("HAVE_SVS=1") - set(svs_factory_file "index_factories/svs_factory.cpp") - set(SVS_TARGET_NAME "svs::svs") if(SVS_SHARED_LIB) include(FetchContent) FetchContent_Declare( @@ -70,7 +65,6 @@ if(USE_SVS) list(APPEND CMAKE_PREFIX_PATH "${svs_SOURCE_DIR}") find_package(svs REQUIRED) set(SVS_LVQ_HEADER "svs/extensions/vamana/lvq.h") - set(SVS_TARGET_NAME "svs::svs_shared_library") else() # This file is included from both CMakeLists.txt and python_bindings/CMakeLists.txt # Set `root` relative to this file, regardless of where it is included from. @@ -80,16 +74,16 @@ if(USE_SVS) deps/ScalableVectorSearch ) set(SVS_LVQ_HEADER "svs/quantization/lvq/impl/lvq_impl.h") - set(SVS_TARGET_NAME "svs::svs") endif() -else() - add_compile_definitions("HAVE_SVS=0") -endif() -if(EXISTS "${svs_SOURCE_DIR}/include/${SVS_LVQ_HEADER}") - message("SVS LVQ implementation found") - add_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=1" PUBLIC "SVS_LVQ_HEADER=\"${SVS_LVQ_HEADER}\"") + if(EXISTS "${svs_SOURCE_DIR}/include/${SVS_LVQ_HEADER}") + message("SVS LVQ implementation found") + add_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=1" PUBLIC "SVS_LVQ_HEADER=\"${SVS_LVQ_HEADER}\"") + else() + message("SVS LVQ implementation not found") + add_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=0") + endif() else() - message("SVS LVQ implementation not found") - add_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=0") + message(STATUS "SVS support disabled") + add_compile_definitions("HAVE_SVS=0") endif() From 461e3152da445c09ce6374971c20cf178578fb19 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Thu, 24 Apr 2025 15:47:59 +0200 Subject: [PATCH 30/32] Remove MKL dependency --- .github/workflows/task-unit-test.yml | 2 +- .install/amazon_linux_2.sh | 1 - .install/common_base_linux_mariner_2.0.sh | 1 - .install/debian_gnu_linux_11.sh | 1 - .install/install_mkl.sh | 18 ------------------ .install/install_mkl_apt.sh | 16 ---------------- .install/rocky_linux_8.sh | 1 - .install/rocky_linux_9.sh | 1 - .install/ubuntu_16.04.sh | 1 - .install/ubuntu_18.04.sh | 1 - .install/ubuntu_20.04.sh | 1 - .install/ubuntu_22.04.sh | 1 - .install/ubuntu_24.04.sh | 1 - cmake/svs.cmake | 17 ++--------------- 14 files changed, 3 insertions(+), 60 deletions(-) delete mode 100644 .install/install_mkl.sh delete mode 100644 .install/install_mkl_apt.sh diff --git a/.github/workflows/task-unit-test.yml b/.github/workflows/task-unit-test.yml index edff79b90..f18b2195e 100644 --- a/.github/workflows/task-unit-test.yml +++ b/.github/workflows/task-unit-test.yml @@ -49,7 +49,7 @@ jobs: run: make unit_test - name: valgrind if: ${{ inputs.run-valgrind }} - run: make valgrind CMAKE_FLAGS="-DSVS_SHARED_LIB=OFF" + run: make valgrind - name: Archive valgrind tests reports if: ${{ inputs.run-valgrind && failure() }} uses: actions/upload-artifact@v4 diff --git a/.install/amazon_linux_2.sh b/.install/amazon_linux_2.sh index 9d1b55d08..06ed36887 100644 --- a/.install/amazon_linux_2.sh +++ b/.install/amazon_linux_2.sh @@ -10,4 +10,3 @@ $MODE yum install -y wget git valgrind gcc10 gcc10-c++ $MODE update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc10-gcc 60 --slave /usr/bin/g++ g++ /usr/bin/gcc10-g++ source install_cmake.sh $MODE -source install_mkl.sh $MODE diff --git a/.install/common_base_linux_mariner_2.0.sh b/.install/common_base_linux_mariner_2.0.sh index 82033013c..e928e7e74 100644 --- a/.install/common_base_linux_mariner_2.0.sh +++ b/.install/common_base_linux_mariner_2.0.sh @@ -6,4 +6,3 @@ export DEBIAN_FRONTEND=noninteractive $MODE tdnf install -y build-essential git wget ca-certificates which source install_cmake.sh $MODE -source install_mkl.sh $MODE diff --git a/.install/debian_gnu_linux_11.sh b/.install/debian_gnu_linux_11.sh index 97838cd92..febfb4a44 100644 --- a/.install/debian_gnu_linux_11.sh +++ b/.install/debian_gnu_linux_11.sh @@ -6,4 +6,3 @@ MODE=$1 # whether to install using sudo or not $MODE apt update -qq $MODE apt install -yqq git wget build-essential lcov valgrind source install_cmake.sh $MODE -source install_mkl_apt.sh $MODE diff --git a/.install/install_mkl.sh b/.install/install_mkl.sh deleted file mode 100644 index 522896b01..000000000 --- a/.install/install_mkl.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -filename=l_BaseKit_p_2024.1.0.596.sh -uuid=fdc7a2bc-b7a8-47eb-8876-de6201297144 -prefix=/opt/intel/oneapi -processor=$(uname -m) -OS_TYPE=$(uname -s) -MODE=$1 # whether to install using sudo or not - -if [[ $OS_TYPE = 'Linux' ]] -then - if [[ $processor = 'x86_64' ]] - then - wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/${uuid}/${filename} - chmod u+x ./${filename} - $MODE ./${filename} -a --action install --silent --ignore-errors --eula accept --components intel.oneapi.lin.mkl.devel --install-dir ${prefix} - #source ${prefix}/setvars.sh - fi -fi diff --git a/.install/install_mkl_apt.sh b/.install/install_mkl_apt.sh deleted file mode 100644 index 135524d55..000000000 --- a/.install/install_mkl_apt.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -version=2024.1 -processor=$(uname -m) -OS_TYPE=$(uname -s) -MODE=$1 # whether to install using sudo or not - -if [[ $OS_TYPE = 'Linux' ]] -then - if [[ $processor = 'x86_64' ]] - then - wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | $MODE tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null - echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | $MODE tee /etc/apt/sources.list.d/intel-oneapi.list - $MODE apt update -qq || true - $MODE apt install -yqq intel-oneapi-mkl-core-devel-${version} - fi -fi diff --git a/.install/rocky_linux_8.sh b/.install/rocky_linux_8.sh index 1a5db2143..dbb6cdbb9 100755 --- a/.install/rocky_linux_8.sh +++ b/.install/rocky_linux_8.sh @@ -19,4 +19,3 @@ $MODE dnf install -y gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ gcc-toolset-13-li cp /opt/rh/gcc-toolset-13/enable /etc/profile.d/gcc-toolset-13.sh source install_cmake.sh $MODE -source install_mkl.sh $MODE diff --git a/.install/rocky_linux_9.sh b/.install/rocky_linux_9.sh index 05f83fef5..523998c8c 100644 --- a/.install/rocky_linux_9.sh +++ b/.install/rocky_linux_9.sh @@ -8,4 +8,3 @@ $MODE dnf install -y gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ make wget git val cp /opt/rh/gcc-toolset-13/enable /etc/profile.d/gcc-toolset-13.sh source install_cmake.sh $MODE -source install_mkl.sh $MODE diff --git a/.install/ubuntu_16.04.sh b/.install/ubuntu_16.04.sh index 5368e6dc1..2eef88bb7 100755 --- a/.install/ubuntu_16.04.sh +++ b/.install/ubuntu_16.04.sh @@ -12,4 +12,3 @@ $MODE apt update $MODE apt-get install -yqq git wget make valgrind gcc-9 g++-9 $MODE update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-9 source install_cmake.sh $MODE -source install_mkl_apt.sh $MODE diff --git a/.install/ubuntu_18.04.sh b/.install/ubuntu_18.04.sh index 670875ea2..73390b401 100644 --- a/.install/ubuntu_18.04.sh +++ b/.install/ubuntu_18.04.sh @@ -21,4 +21,3 @@ $MODE make install cd .. source install_cmake.sh $MODE -source install_mkl_apt.sh $MODE diff --git a/.install/ubuntu_20.04.sh b/.install/ubuntu_20.04.sh index 522893f26..6b45d2fe8 100755 --- a/.install/ubuntu_20.04.sh +++ b/.install/ubuntu_20.04.sh @@ -12,4 +12,3 @@ $MODE update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 60 --slave # align gcov version with gcc version $MODE update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-11 60 source install_cmake.sh $MODE -source install_mkl_apt.sh $MODE diff --git a/.install/ubuntu_22.04.sh b/.install/ubuntu_22.04.sh index 8263bd60f..0075f9a1e 100755 --- a/.install/ubuntu_22.04.sh +++ b/.install/ubuntu_22.04.sh @@ -9,4 +9,3 @@ $MODE update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 60 --slave # align gcov version with gcc version $MODE update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-12 60 source install_cmake.sh $MODE -source install_mkl_apt.sh $MODE diff --git a/.install/ubuntu_24.04.sh b/.install/ubuntu_24.04.sh index 13234ea9b..4c06a346a 100755 --- a/.install/ubuntu_24.04.sh +++ b/.install/ubuntu_24.04.sh @@ -7,4 +7,3 @@ $MODE apt update -qq $MODE apt install -yqq git wget build-essential lcov openssl libssl-dev \ python3 python3-venv python3-dev unzip rsync clang curl source install_cmake.sh $MODE -source install_mkl_apt.sh $MODE diff --git a/cmake/svs.cmake b/cmake/svs.cmake index e367d29ad..72f07512b 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -39,21 +39,8 @@ if(USE_SVS) add_compile_definitions("HAVE_SVS=1") set(svs_factory_file "index_factories/svs_factory.cpp") - # try to find MKL - set(MKL_HINTS ${MKLROOT} $ENV{MKL_DIR} $ENV{MKL_ROOT} $ENV{MKLROOT}) - if(LINUX) - list(APPEND MKL_HINTS "/opt/intel/oneapi/mkl/latest") - endif() - if(WIN32) - list(APPEND MKL_HINTS "C:/Program Files (x86)/Intel/oneAPI/mkl/latest") - endif() - find_package(MKL HINTS ${MKL_HINTS}) - if(NOT MKL_FOUND) - message(WARNING "MKL not found - disabling SVS shared library") - endif() - - cmake_dependent_option(SVS_SHARED_LIB "Use SVS pre-compiled shared library" ON "USE_SVS AND MKL_FOUND" OFF) - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.7/svs-shared-library-0.0.7-avx2.tar.gz" CACHE STRING "SVS URL") + cmake_dependent_option(SVS_SHARED_LIB "Use SVS pre-compiled shared library" ON "USE_SVS" OFF) + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.8-dev/svs-shared-library-0.0.8-NIGHTLY-20250423.tar.gz" CACHE STRING "SVS URL") if(SVS_SHARED_LIB) include(FetchContent) From 7a4322778568eea3b9758ad7be9f2cec9537fee8 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Fri, 25 Apr 2025 17:03:57 +0200 Subject: [PATCH 31/32] Add glibc detection for SVS shared library --- cmake/svs.cmake | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index 72f07512b..edcf273b2 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -39,7 +39,15 @@ if(USE_SVS) add_compile_definitions("HAVE_SVS=1") set(svs_factory_file "index_factories/svs_factory.cpp") - cmake_dependent_option(SVS_SHARED_LIB "Use SVS pre-compiled shared library" ON "USE_SVS" OFF) + # detect if build environment is using glibc + include(CheckSymbolExists) + unset(GLIBC_FOUND CACHE) + check_symbol_exists(__GLIBC__ "features.h" GLIBC_FOUND) + if(NOT GLIBC_FOUND) + message(STATUS "GLIBC is not detected - SVS shared library is not supported") + endif() + + cmake_dependent_option(SVS_SHARED_LIB "Use SVS pre-compiled shared library" ON "USE_SVS AND GLIBC_FOUND" OFF) set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.8-dev/svs-shared-library-0.0.8-NIGHTLY-20250423.tar.gz" CACHE STRING "SVS URL") if(SVS_SHARED_LIB) From 35cd71cd76ce1c143619beb631089be0da6ebd49 Mon Sep 17 00:00:00 2001 From: Rafik Saliev Date: Mon, 28 Apr 2025 10:44:43 +0200 Subject: [PATCH 32/32] Code review changes s1e1 --- cmake/svs.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index edcf273b2..3270c14d2 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -8,6 +8,7 @@ endif() # so disable it if we are in Debug mode string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) if(uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG") + message(STATUS "SVS: Disabling AVX512 support in Debug mode due to Valgrind") set(SVS_NO_AVX512 ON) endif() @@ -23,7 +24,7 @@ endif() if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11.0") set(SVS_SUPPORTED 0) - message(STATUS "Insufficient gcc version for SVS") + message(STATUS "Skipping SVS: requires GCC >= 11.") endif() endif()