[MOD-9384] [SVS] Fix CI builds (#658)

* [SVS] Fix CI builds

* Add CPU detection for SVS
* Add MKL installation for SVS shared library

* Extend MKL package lookup in CMAKE

* Change MKL version to 2024.1

* Add --ignore-errors to MKL installer

* Add GCC version check for SVS

* Fix Alpine3 compilation error

* Enable SVS_SHARED_LIB ON

* Install MKL on Ubuntu/Debian via APT

* Reduce MKL size in Ubuntu by using 'core' package.

* durtyfix svs binary package issue

* Disable SVS share lib if MKL not found

* Revert "durtyfix svs binary package issue"

This reverts commit 1a10a0a.

* fixup! Disable SVS share lib if MKL not found

* fixup! Refactor Info Report - [MOD-9321] (#650)

* Revert SVS disabling changes made in f670214

* Move SVS cmake checks and definitions to a separate file.

* Disable SVS AVX512 for Valgrind Debug build

* Fix Valgrind tests

* Disable SVS shared library for Valgrind tests

* Update svs_allocator.patch

* Disable SVS shared library by default

* Add 'patch' tool installation required for SVS  to rocky9

* Revert "Disable SVS shared library by default"

This reverts commit 7b9c5ad.

* Update SVS submodule and remove allocator patch

* enable mac tests
disable skipping

* Revert "enable mac tests"

This reverts commit 79c5682.

* Address code review comments

* Address code review comments s1e2

* Move all SVS configuration under one if

* Remove MKL dependency

* Add glibc detection for SVS shared library

* Code review changes s1e1

---------

Co-authored-by: meiravgri <[email protected]>

Author: rfsaliev

.install/alpine_linux_3.sh

@@ -4,6 +4,6 @@ set -e
 
 $MODE apk update
 
-$MODE apk add --no-cache build-base gcc g++ make wget git valgrind
+$MODE apk add --no-cache build-base gcc g++ make wget git valgrind linux-headers
 
 $MODE apk add --no-cache cmake

CMakeLists.txt

@@ -33,6 +33,8 @@ IF(USE_PROFILE)
 	SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
 ENDIF()
 
+include(cmake/svs.cmake)
+
 include_directories(src)
 
 if(VECSIM_BUILD_TESTS)

Makefile

@@ -197,7 +197,7 @@ endif
 
 flow_test:
 	$(SHOW)poetry install $(POETRY_ARGS)
-	$(SHOW)poetry run pytest tests/flow/$(TEST) --deselect=tests/flow/test_svs.py -v -s
+	$(SHOW)poetry run pytest tests/flow/$(TEST) -v -s
 
 .PHONY: flow_test
 

cmake/svs.cmake

@@ -0,0 +1,85 @@
+cmake_minimum_required(VERSION 3.12)
+cmake_policy(SET CMP0074 NEW)
+if(POLICY CMP0135)
+    cmake_policy(SET CMP0135 NEW)
+endif()
+
+# Valgrind does not support AVX512 and Valgrind in running in Debug
+# so disable it if we are in Debug mode
+string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE)
+if(uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+    message(STATUS "SVS: Disabling AVX512 support in Debug mode due to Valgrind")
+    set(SVS_NO_AVX512 ON)
+endif()
+
+if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "(x86_64)|(AMD64|amd64)")
+    set(SVS_SUPPORTED 1)
+else()
+    set(SVS_SUPPORTED 0)
+    message(STATUS "SVS is not supported on this architecture")
+endif()
+
+# GCC < v11 does not support C++20 features required for SVS
+# https://gcc.gnu.org/projects/cxx-status.html
+if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11.0")
+        set(SVS_SUPPORTED 0)
+        message(STATUS "Skipping SVS: requires GCC >= 11.")
+    endif()
+endif()
+
+include(CMakeDependentOption)
+
+# USE_SVS option forcibly OFF if CPU or compiler is not supported
+# elsewhere let user disable SVS
+cmake_dependent_option(USE_SVS "Build with SVS library support" ON "SVS_SUPPORTED" OFF)
+
+if(USE_SVS)
+    message(STATUS "SVS support enabled")
+    # Configure SVS build
+    add_compile_definitions("HAVE_SVS=1")
+    set(svs_factory_file "index_factories/svs_factory.cpp")
+
+    # detect if build environment is using glibc
+    include(CheckSymbolExists)
+    unset(GLIBC_FOUND CACHE)
+    check_symbol_exists(__GLIBC__ "features.h" GLIBC_FOUND)
+    if(NOT GLIBC_FOUND)
+        message(STATUS "GLIBC is not detected - SVS shared library is not supported")
+    endif()
+
+    cmake_dependent_option(SVS_SHARED_LIB "Use SVS pre-compiled shared library" ON "USE_SVS AND GLIBC_FOUND" OFF)
+    set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.8-dev/svs-shared-library-0.0.8-NIGHTLY-20250423.tar.gz" CACHE STRING "SVS URL")
+
+    if(SVS_SHARED_LIB)
+        include(FetchContent)
+        FetchContent_Declare(
+            svs
+            URL "${SVS_URL}"
+        )
+        FetchContent_MakeAvailable(svs)
+        list(APPEND CMAKE_PREFIX_PATH "${svs_SOURCE_DIR}")
+        find_package(svs REQUIRED)
+        set(SVS_LVQ_HEADER "svs/extensions/vamana/lvq.h")
+    else()
+        # This file is included from both CMakeLists.txt and python_bindings/CMakeLists.txt  
+        # Set `root` relative to this file, regardless of where it is included from. 
+        get_filename_component(root ${CMAKE_CURRENT_LIST_DIR}/.. ABSOLUTE)
+        add_subdirectory(
+            ${root}/deps/ScalableVectorSearch
+            deps/ScalableVectorSearch
+        )
+        set(SVS_LVQ_HEADER "svs/quantization/lvq/impl/lvq_impl.h")
+    endif()
+
+    if(EXISTS "${svs_SOURCE_DIR}/include/${SVS_LVQ_HEADER}")
+        message("SVS LVQ implementation found")
+        add_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=1" PUBLIC "SVS_LVQ_HEADER=\"${SVS_LVQ_HEADER}\"")
+    else()
+        message("SVS LVQ implementation not found")
+        add_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=0")
+    endif()
+else()
+    message(STATUS "SVS support disabled")
+    add_compile_definitions("HAVE_SVS=0")
+endif()

deps/ScalableVectorSearch

@@ -17,46 +17,14 @@ set(CMAKE_CXX_FLAGS_DEBUG "-g")
 set(CMAKE_CXX_FLAGS_RELEASE "-O3")
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g")
 
-# option(SVS_EXPERIMENTAL_LEANVEC "Enable experimental LeanVec in SVS" YES)
-# if(SVS_EXPERIMENTAL_LEANVEC)
-#     add_definitions(-DSVS_EXPERIMENTAL_LEANVEC)
-# endif()
-
-# TODO: enable svs build again once MKL installation issue is resolved, and after we validate
-# that we only build for supported platforms (and fail gracefully for those that are not)
-option(SVS_SUPPORTED "Build with SVS library support" OFF)
+add_subdirectory(spaces)
 
-# Configure SVS build
-set(SVS_SHARED_LIB ON CACHE BOOL "Use SVS pre-compiled shared library")
-set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.7/svs-shared-library-0.0.7-avx2.tar.gz" CACHE STRING "SVS URL")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall")
 
-if(SVS_SUPPORTED)
+if(USE_SVS)
     set(svs_factory_file "index_factories/svs_factory.cpp")
-    if(SVS_SHARED_LIB)
-        include(FetchContent)
-        FetchContent_Declare(
-            svs
-            URL "${SVS_URL}"
-        )
-        FetchContent_MakeAvailable(svs)
-        list(APPEND CMAKE_PREFIX_PATH "${svs_SOURCE_DIR}")
-        find_package(MKL REQUIRED)
-        find_package(svs REQUIRED)
-        set(SVS_LVQ_HEADER "svs/extensions/vamana/lvq.h")
-    else()
-        get_filename_component(root ${CMAKE_CURRENT_LIST_DIR}/../.. ABSOLUTE)
-        add_subdirectory(
-            ${root}/deps/ScalableVectorSearch
-            deps/ScalableVectorSearch
-        )
-        set(SVS_LVQ_HEADER "svs/quantization/lvq/impl/lvq_impl.h")
-    endif()
 endif()
 
-add_subdirectory(spaces)
-
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall")
-
 add_library(VectorSimilarity ${VECSIM_LIBTYPE}
     index_factories/brute_force_factory.cpp
     index_factories/hnsw_factory.cpp
@@ -78,23 +46,13 @@ add_library(VectorSimilarity ${VECSIM_LIBTYPE}
     ${HEADER_LIST}
 )
 
-if (SVS_SUPPORTED)
-    target_link_libraries(VectorSimilarity VectorSimilaritySpaces svs::svs)
+target_link_libraries(VectorSimilarity VectorSimilaritySpaces)
+
+if (TARGET svs::svs)
+    target_link_libraries(VectorSimilarity svs::svs)
     if(TARGET svs::svs_shared_library)
-        target_link_libraries(VectorSimilarity svs::svs_shared_library MKL::MKL)
+        target_link_libraries(VectorSimilarity svs::svs_shared_library)
     endif()
-    target_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS=1")
-else()
-    target_link_libraries(VectorSimilarity VectorSimilaritySpaces)
-    target_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS=0")
-endif()
-
-if(EXISTS "${svs_SOURCE_DIR}/include/${SVS_LVQ_HEADER}")
-    message("SVS LVQ implementation found")
-    target_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=1" PUBLIC "SVS_LVQ_HEADER=\"${SVS_LVQ_HEADER}\"")
-else()
-    message("SVS LVQ implementation not found")
-    target_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=0")
 endif()
 
 if(VECSIM_BUILD_TESTS)

src/VecSim/CMakeLists.txt

@@ -258,7 +258,7 @@ class SVSIndex : public VecSimIndexAbstract<svs_details::vecsim_dt<DataType>, fl
         return info;
     }
 
-    VecSimIndexDInfo debugInfo() const override {
+    VecSimIndexDebugInfo debugInfo() const override {
         VecSimIndexDebugInfo info;
         info.commonInfo = this->getCommonInfo();
         info.commonInfo.basicInfo.algo = VecSimAlgo_SVS;

src/VecSim/algorithms/svs/svs.h

@@ -20,6 +20,7 @@ if(NOT pybind11_POPULATED)
     add_subdirectory(${pybind11_SOURCE_DIR} ${pybind11_BINARY_DIR})
 endif()
 
+include(${root}/cmake/svs.cmake)
 add_subdirectory(${root}/src/VecSim VectorSimilarity)
 
 include_directories(${root}/src ${root}/tests/utils)

src/python_bindings/CMakeLists.txt

@@ -60,6 +60,11 @@ target_link_libraries(test_fp16 PUBLIC gtest_main VectorSimilarity)
 target_link_libraries(test_int8 PUBLIC gtest_main VectorSimilarity)
 target_link_libraries(test_uint8 PUBLIC gtest_main VectorSimilarity)
 
+if(USE_SVS)
+	add_executable(test_svs ../utils/mock_thread_pool.cpp test_svs.cpp unit_test_utils.cpp)
+	target_link_libraries(test_svs PUBLIC gtest_main VectorSimilarity)
+endif()
+
 include(GoogleTest)
 
 gtest_discover_tests(test_hnsw)
@@ -75,8 +80,6 @@ gtest_discover_tests(test_fp16 TEST_PREFIX FP16UNIT_)
 gtest_discover_tests(test_int8 TEST_PREFIX INT8UNIT_)
 gtest_discover_tests(test_uint8 TEST_PREFIX UINT8UNIT_)
 
-if(SVS_SUPPORTED)
-	add_executable(test_svs ../utils/mock_thread_pool.cpp test_svs.cpp unit_test_utils.cpp)
-	target_link_libraries(test_svs PUBLIC gtest_main VectorSimilarity)
+if(USE_SVS)
 	gtest_discover_tests(test_svs)
-endif()
+endif()

tests/unit/CMakeLists.txt

@@ -484,7 +484,7 @@ TYPED_TEST(SVSTest, svs_batch_iterator_non_unique_scores) {
     // as the number of results is 5, which is more than 0.1% of the index size. for index of size
     // 10000, we will run the heap-based search until we return 5000 results, and then switch to
     // select-based search.
-    for (size_t n : {100, 10000}) {
+    for (size_t n : {100, 1000}) {
         SVSParams params = {
             .dim = dim,
             .metric = VecSimMetric_L2,
@@ -1178,13 +1178,6 @@ TYPED_TEST(SVSTest, svs_test_inf_score) {
         .dim = dim,
         .metric = VecSimMetric_L2,
         .blockSize = 1,
-        /* SVS-Vamana specifics */
-        .alpha = 1.2,
-        .graph_max_degree = 64,
-        .construction_window_size = 20,
-        .max_candidate_pool_size = 1024,
-        .prune_to = 60,
-        .use_search_history = VecSimOption_ENABLE,
     };
 
     VecSimIndex *index = this->CreateNewIndex(params);