diff --git a/.install/alpine_linux_3.sh b/.install/alpine_linux_3.sh index 4766d44ee..5fc4bed04 100644 --- a/.install/alpine_linux_3.sh +++ b/.install/alpine_linux_3.sh @@ -4,6 +4,6 @@ set -e $MODE apk update -$MODE apk add --no-cache build-base gcc g++ make wget git valgrind +$MODE apk add --no-cache build-base gcc g++ make wget git valgrind linux-headers $MODE apk add --no-cache cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index c52d42b84..e3e36de25 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,8 @@ IF(USE_PROFILE) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") ENDIF() +include(cmake/svs.cmake) + include_directories(src) if(VECSIM_BUILD_TESTS) diff --git a/Makefile b/Makefile index 832a03849..450194de8 100644 --- a/Makefile +++ b/Makefile @@ -197,7 +197,7 @@ endif flow_test: $(SHOW)poetry install $(POETRY_ARGS) - $(SHOW)poetry run pytest tests/flow/$(TEST) --deselect=tests/flow/test_svs.py -v -s + $(SHOW)poetry run pytest tests/flow/$(TEST) -v -s .PHONY: flow_test diff --git a/cmake/svs.cmake b/cmake/svs.cmake new file mode 100644 index 000000000..3270c14d2 --- /dev/null +++ b/cmake/svs.cmake @@ -0,0 +1,85 @@ +cmake_minimum_required(VERSION 3.12) +cmake_policy(SET CMP0074 NEW) +if(POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) +endif() + +# Valgrind does not support AVX512 and Valgrind in running in Debug +# so disable it if we are in Debug mode +string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) +if(uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG") + message(STATUS "SVS: Disabling AVX512 support in Debug mode due to Valgrind") + set(SVS_NO_AVX512 ON) +endif() + +if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "(x86_64)|(AMD64|amd64)") + set(SVS_SUPPORTED 1) +else() + set(SVS_SUPPORTED 0) + message(STATUS "SVS is not supported on this architecture") +endif() + +# GCC < v11 does not support C++20 features required for SVS +# https://gcc.gnu.org/projects/cxx-status.html +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11.0") + set(SVS_SUPPORTED 0) + message(STATUS "Skipping SVS: requires GCC >= 11.") + endif() +endif() + +include(CMakeDependentOption) + +# USE_SVS option forcibly OFF if CPU or compiler is not supported +# elsewhere let user disable SVS +cmake_dependent_option(USE_SVS "Build with SVS library support" ON "SVS_SUPPORTED" OFF) + +if(USE_SVS) + message(STATUS "SVS support enabled") + # Configure SVS build + add_compile_definitions("HAVE_SVS=1") + set(svs_factory_file "index_factories/svs_factory.cpp") + + # detect if build environment is using glibc + include(CheckSymbolExists) + unset(GLIBC_FOUND CACHE) + check_symbol_exists(__GLIBC__ "features.h" GLIBC_FOUND) + if(NOT GLIBC_FOUND) + message(STATUS "GLIBC is not detected - SVS shared library is not supported") + endif() + + cmake_dependent_option(SVS_SHARED_LIB "Use SVS pre-compiled shared library" ON "USE_SVS AND GLIBC_FOUND" OFF) + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.8-dev/svs-shared-library-0.0.8-NIGHTLY-20250423.tar.gz" CACHE STRING "SVS URL") + + if(SVS_SHARED_LIB) + include(FetchContent) + FetchContent_Declare( + svs + URL "${SVS_URL}" + ) + FetchContent_MakeAvailable(svs) + list(APPEND CMAKE_PREFIX_PATH "${svs_SOURCE_DIR}") + find_package(svs REQUIRED) + set(SVS_LVQ_HEADER "svs/extensions/vamana/lvq.h") + else() + # This file is included from both CMakeLists.txt and python_bindings/CMakeLists.txt + # Set `root` relative to this file, regardless of where it is included from. + get_filename_component(root ${CMAKE_CURRENT_LIST_DIR}/.. ABSOLUTE) + add_subdirectory( + ${root}/deps/ScalableVectorSearch + deps/ScalableVectorSearch + ) + set(SVS_LVQ_HEADER "svs/quantization/lvq/impl/lvq_impl.h") + endif() + + if(EXISTS "${svs_SOURCE_DIR}/include/${SVS_LVQ_HEADER}") + message("SVS LVQ implementation found") + add_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=1" PUBLIC "SVS_LVQ_HEADER=\"${SVS_LVQ_HEADER}\"") + else() + message("SVS LVQ implementation not found") + add_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=0") + endif() +else() + message(STATUS "SVS support disabled") + add_compile_definitions("HAVE_SVS=0") +endif() diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch index 1e59bf6a9..5a7da989d 160000 --- a/deps/ScalableVectorSearch +++ b/deps/ScalableVectorSearch @@ -1 +1 @@ -Subproject commit 1e59bf6a9db91886dbded8bf18e15fb1ad8e3e44 +Subproject commit 5a7da989d0d0e5a603851069c90465498845a78b diff --git a/src/VecSim/CMakeLists.txt b/src/VecSim/CMakeLists.txt index 58d45f203..cb23e46fe 100644 --- a/src/VecSim/CMakeLists.txt +++ b/src/VecSim/CMakeLists.txt @@ -17,46 +17,14 @@ set(CMAKE_CXX_FLAGS_DEBUG "-g") set(CMAKE_CXX_FLAGS_RELEASE "-O3") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g") -# option(SVS_EXPERIMENTAL_LEANVEC "Enable experimental LeanVec in SVS" YES) -# if(SVS_EXPERIMENTAL_LEANVEC) -# add_definitions(-DSVS_EXPERIMENTAL_LEANVEC) -# endif() - -# TODO: enable svs build again once MKL installation issue is resolved, and after we validate -# that we only build for supported platforms (and fail gracefully for those that are not) -option(SVS_SUPPORTED "Build with SVS library support" OFF) +add_subdirectory(spaces) -# Configure SVS build -set(SVS_SHARED_LIB ON CACHE BOOL "Use SVS pre-compiled shared library") -set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.7/svs-shared-library-0.0.7-avx2.tar.gz" CACHE STRING "SVS URL") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall") -if(SVS_SUPPORTED) +if(USE_SVS) set(svs_factory_file "index_factories/svs_factory.cpp") - if(SVS_SHARED_LIB) - include(FetchContent) - FetchContent_Declare( - svs - URL "${SVS_URL}" - ) - FetchContent_MakeAvailable(svs) - list(APPEND CMAKE_PREFIX_PATH "${svs_SOURCE_DIR}") - find_package(MKL REQUIRED) - find_package(svs REQUIRED) - set(SVS_LVQ_HEADER "svs/extensions/vamana/lvq.h") - else() - get_filename_component(root ${CMAKE_CURRENT_LIST_DIR}/../.. ABSOLUTE) - add_subdirectory( - ${root}/deps/ScalableVectorSearch - deps/ScalableVectorSearch - ) - set(SVS_LVQ_HEADER "svs/quantization/lvq/impl/lvq_impl.h") - endif() endif() -add_subdirectory(spaces) - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall") - add_library(VectorSimilarity ${VECSIM_LIBTYPE} index_factories/brute_force_factory.cpp index_factories/hnsw_factory.cpp @@ -78,23 +46,13 @@ add_library(VectorSimilarity ${VECSIM_LIBTYPE} ${HEADER_LIST} ) -if (SVS_SUPPORTED) - target_link_libraries(VectorSimilarity VectorSimilaritySpaces svs::svs) +target_link_libraries(VectorSimilarity VectorSimilaritySpaces) + +if (TARGET svs::svs) + target_link_libraries(VectorSimilarity svs::svs) if(TARGET svs::svs_shared_library) - target_link_libraries(VectorSimilarity svs::svs_shared_library MKL::MKL) + target_link_libraries(VectorSimilarity svs::svs_shared_library) endif() - target_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS=1") -else() - target_link_libraries(VectorSimilarity VectorSimilaritySpaces) - target_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS=0") -endif() - -if(EXISTS "${svs_SOURCE_DIR}/include/${SVS_LVQ_HEADER}") - message("SVS LVQ implementation found") - target_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=1" PUBLIC "SVS_LVQ_HEADER=\"${SVS_LVQ_HEADER}\"") -else() - message("SVS LVQ implementation not found") - target_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=0") endif() if(VECSIM_BUILD_TESTS) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 6210f6e83..4de31fb8c 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -258,7 +258,7 @@ class SVSIndex : public VecSimIndexAbstract, fl return info; } - VecSimIndexDInfo debugInfo() const override { + VecSimIndexDebugInfo debugInfo() const override { VecSimIndexDebugInfo info; info.commonInfo = this->getCommonInfo(); info.commonInfo.basicInfo.algo = VecSimAlgo_SVS; diff --git a/src/python_bindings/CMakeLists.txt b/src/python_bindings/CMakeLists.txt index de51f0a27..48d4a71fa 100644 --- a/src/python_bindings/CMakeLists.txt +++ b/src/python_bindings/CMakeLists.txt @@ -20,6 +20,7 @@ if(NOT pybind11_POPULATED) add_subdirectory(${pybind11_SOURCE_DIR} ${pybind11_BINARY_DIR}) endif() +include(${root}/cmake/svs.cmake) add_subdirectory(${root}/src/VecSim VectorSimilarity) include_directories(${root}/src ${root}/tests/utils) diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index f248705d4..7b1171faf 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -60,6 +60,11 @@ target_link_libraries(test_fp16 PUBLIC gtest_main VectorSimilarity) target_link_libraries(test_int8 PUBLIC gtest_main VectorSimilarity) target_link_libraries(test_uint8 PUBLIC gtest_main VectorSimilarity) +if(USE_SVS) + add_executable(test_svs ../utils/mock_thread_pool.cpp test_svs.cpp unit_test_utils.cpp) + target_link_libraries(test_svs PUBLIC gtest_main VectorSimilarity) +endif() + include(GoogleTest) gtest_discover_tests(test_hnsw) @@ -75,8 +80,6 @@ gtest_discover_tests(test_fp16 TEST_PREFIX FP16UNIT_) gtest_discover_tests(test_int8 TEST_PREFIX INT8UNIT_) gtest_discover_tests(test_uint8 TEST_PREFIX UINT8UNIT_) -if(SVS_SUPPORTED) - add_executable(test_svs ../utils/mock_thread_pool.cpp test_svs.cpp unit_test_utils.cpp) - target_link_libraries(test_svs PUBLIC gtest_main VectorSimilarity) +if(USE_SVS) gtest_discover_tests(test_svs) -endif() \ No newline at end of file +endif() diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index 1f12ec0a6..1706040ff 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -484,7 +484,7 @@ TYPED_TEST(SVSTest, svs_batch_iterator_non_unique_scores) { // as the number of results is 5, which is more than 0.1% of the index size. for index of size // 10000, we will run the heap-based search until we return 5000 results, and then switch to // select-based search. - for (size_t n : {100, 10000}) { + for (size_t n : {100, 1000}) { SVSParams params = { .dim = dim, .metric = VecSimMetric_L2, @@ -1178,13 +1178,6 @@ TYPED_TEST(SVSTest, svs_test_inf_score) { .dim = dim, .metric = VecSimMetric_L2, .blockSize = 1, - /* SVS-Vamana specifics */ - .alpha = 1.2, - .graph_max_degree = 64, - .construction_window_size = 20, - .max_candidate_pool_size = 1024, - .prune_to = 60, - .use_search_history = VecSimOption_ENABLE, }; VecSimIndex *index = this->CreateNewIndex(params);