From 7944d8f3a801b2a08c3bbb5bea8b7cff6a1395f3 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 5 Feb 2021 12:05:01 +0000 Subject: [PATCH 1/8] WIP: Hack to optimise based on known sizes If the caller knows the sizes, and knows the allocation is thread local, then we can make some significant optimisations. This is a brief hack to show where would need changing. --- src/mem/alloc.h | 25 ++++++++++++++++++++++++- src/mem/sizeclass.h | 2 +- src/mem/sizeclasstable.h | 3 ++- src/override/malloc.cc | 19 +++++++++++++++++++ src/test/func/malloc/malloc.cc | 11 +++++++++++ 5 files changed, 57 insertions(+), 3 deletions(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 6ec2f8bae..9496c834f 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -1040,7 +1040,8 @@ namespace snmalloc allow_reserve == NoReserve ? "noreserve" : "reserve")); SNMALLOC_ASSUME(size <= SLAB_SIZE); - sizeclass_t sizeclass = size_to_sizeclass(size); + SNMALLOC_ASSUME(size > 0); + sizeclass_t sizeclass = size_to_sizeclass(size); return small_alloc_inner(sizeclass, size); } @@ -1066,6 +1067,13 @@ namespace snmalloc return p; } + return small_alloc_inner_slow(sizeclass, size); + } + + template + SNMALLOC_FAST_PATH void* + small_alloc_inner_slow(sizeclass_t sizeclass, size_t size) + { if (likely(!has_messages())) return small_alloc_next_free_list( sizeclass, size); @@ -1228,6 +1236,21 @@ namespace snmalloc small_dealloc_offseted_inner(super, p, sizeclass); } + static SNMALLOC_FAST_PATH bool small_local_dealloc(void* p) + { + auto super = Superslab::get(p); + Slab* slab = Metaslab::get_slab(p); + return (likely(slab->dealloc_fast(super, p))); + } + + SNMALLOC_FAST_PATH void small_local_dealloc_slow(void* p) + { + auto super = Superslab::get(p); + Slab* slab = Metaslab::get_slab(p); + small_dealloc_offseted_slow(super, p, slab->get_meta().sizeclass); + } + + SNMALLOC_FAST_PATH void small_dealloc_offseted_inner( Superslab* super, void* p, sizeclass_t sizeclass) { diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index 0d39db33f..adaf29d05 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -20,7 +20,7 @@ namespace snmalloc constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc); constexpr static uint16_t medium_slab_free(sizeclass_t sizeclass); - static sizeclass_t size_to_sizeclass(size_t size); + template static sizeclass_t size_to_sizeclass(size_t size); constexpr static inline sizeclass_t size_to_sizeclass_const(size_t size) { diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 5e067db42..2719a1a3d 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -110,9 +110,10 @@ namespace snmalloc return sizeclass_metadata.inverse_cache_friendly_mask[sizeclass]; } + template static inline sizeclass_t size_to_sizeclass(size_t size) { - if ((size - 1) <= (SLAB_SIZE - 1)) + if ((size - 1) <= (SLAB_SIZE - 1) || ASSUME_SMALL) { auto index = sizeclass_lookup_index(size); SNMALLOC_ASSUME(index <= sizeclass_lookup_index(SLAB_SIZE)); diff --git a/src/override/malloc.cc b/src/override/malloc.cc index dc9f7a4ec..0712c8c5b 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -69,6 +69,25 @@ extern "C" return ThreadAlloc::get_noncachable()->alloc(sz); } + SNMALLOC_EXPORT + void SNMALLOC_NAME_MANGLE(free_local_small)(void* ptr) + { + if (Alloc::small_local_dealloc(ptr)) return; + ThreadAlloc::get_noncachable()->small_local_dealloc_slow(ptr); + } + + SNMALLOC_EXPORT + void* SNMALLOC_NAME_MANGLE(malloc_small)(size_t size) + { + return ThreadAlloc::get_noncachable()->small_alloc(size); + } + + SNMALLOC_EXPORT + void* SNMALLOC_NAME_MANGLE(malloc_small_64)() + { + return ThreadAlloc::get_noncachable()->small_alloc(64); + } + SNMALLOC_EXPORT size_t SNMALLOC_NAME_MANGLE(malloc_usable_size)( MALLOC_USABLE_SIZE_QUALIFIER void* ptr) diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index b206cd599..164a7fb7f 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -116,6 +116,12 @@ void test_memalign(size_t size, size_t align, int err, bool null) check_result(size, align, p, err, null); } +void test_local(size_t size) +{ + for (int n = 0; n < 1000; n++) + our_free_local_small(our_malloc_small(size)); +} + int main(int argc, char** argv) { UNUSED(argc); @@ -125,6 +131,11 @@ int main(int argc, char** argv) constexpr int SUCCESS = 0; + for (size_t i = 1; i < SLAB_SIZE; i+=16) + { + test_local(i); + } + test_realloc(our_malloc(64), 4194304, SUCCESS, false); for (sizeclass_t sc = 0; sc < (SUPERSLAB_BITS + 4); sc++) From 44374b587d9ad7c560a080ef0f85391ebcb40061 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 5 Feb 2021 16:27:55 +0000 Subject: [PATCH 2/8] Add a redirection layer for small alloc functions. --- CMakeLists.txt | 13 +++++++++++++ src/mem/alloc.h | 1 - src/mem/sizeclass.h | 3 ++- src/mem/sizeclasstable.h | 2 +- src/override/malloc.cc | 12 ++++++++++-- src/test/func/malloc/malloc.cc | 2 +- 6 files changed, 27 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 24a6d53c3..ee809b413 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,6 +169,19 @@ if(CONST_QUALIFIED_MALLOC_USABLE_SIZE) endif() +# Build a redirection layer for all sizes that are a multiple of +# 16bytes up to 1024. +add_executable(generate src/redirect/generate.cc) +target_link_libraries(generate snmalloc_lib) +add_custom_target(generated ALL + COMMAND generate ${CMAKE_CURRENT_BINARY_DIR}/generated.cc + BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/generated.cc + ) +add_library(redirect_small STATIC src/redirect/redirect.cc) +target_link_libraries(redirect_small snmalloc_lib) +target_include_directories(redirect_small PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + + # To build with just the header library target define SNMALLOC_ONLY_HEADER_LIBRARY # in containing Cmake file. if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 9496c834f..9faae0f05 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -1250,7 +1250,6 @@ namespace snmalloc small_dealloc_offseted_slow(super, p, slab->get_meta().sizeclass); } - SNMALLOC_FAST_PATH void small_dealloc_offseted_inner( Superslab* super, void* p, sizeclass_t sizeclass) { diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index adaf29d05..766097b88 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -20,7 +20,8 @@ namespace snmalloc constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc); constexpr static uint16_t medium_slab_free(sizeclass_t sizeclass); - template static sizeclass_t size_to_sizeclass(size_t size); + template + static sizeclass_t size_to_sizeclass(size_t size); constexpr static inline sizeclass_t size_to_sizeclass_const(size_t size) { diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 2719a1a3d..830b534e2 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -110,7 +110,7 @@ namespace snmalloc return sizeclass_metadata.inverse_cache_friendly_mask[sizeclass]; } - template + template static inline sizeclass_t size_to_sizeclass(size_t size) { if ((size - 1) <= (SLAB_SIZE - 1) || ASSUME_SMALL) diff --git a/src/override/malloc.cc b/src/override/malloc.cc index 0712c8c5b..2c54455e1 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -72,14 +72,16 @@ extern "C" SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(free_local_small)(void* ptr) { - if (Alloc::small_local_dealloc(ptr)) return; + if (Alloc::small_local_dealloc(ptr)) + return; ThreadAlloc::get_noncachable()->small_local_dealloc_slow(ptr); } SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(malloc_small)(size_t size) { - return ThreadAlloc::get_noncachable()->small_alloc(size); + return ThreadAlloc::get_noncachable()->small_alloc( + size); } SNMALLOC_EXPORT @@ -88,6 +90,12 @@ extern "C" return ThreadAlloc::get_noncachable()->small_alloc(64); } + SNMALLOC_EXPORT + void* SNMALLOC_NAME_MANGLE(malloc_small_63)() + { + return ThreadAlloc::get_noncachable()->small_alloc(63); + } + SNMALLOC_EXPORT size_t SNMALLOC_NAME_MANGLE(malloc_usable_size)( MALLOC_USABLE_SIZE_QUALIFIER void* ptr) diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 164a7fb7f..e935068cd 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -131,7 +131,7 @@ int main(int argc, char** argv) constexpr int SUCCESS = 0; - for (size_t i = 1; i < SLAB_SIZE; i+=16) + for (size_t i = 1; i < SLAB_SIZE; i += 16) { test_local(i); } From 8b294b27ebd4f6568bfdd1a5bbd8fd1cb1b4bbdb Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 5 Feb 2021 20:17:51 +0000 Subject: [PATCH 3/8] Add missing files.2 --- src/redirect/generate.cc | 35 +++++++++++++++++++++++++++++++++++ src/redirect/redirect.cc | 23 +++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 src/redirect/generate.cc create mode 100644 src/redirect/redirect.cc diff --git a/src/redirect/generate.cc b/src/redirect/generate.cc new file mode 100644 index 000000000..e190aee07 --- /dev/null +++ b/src/redirect/generate.cc @@ -0,0 +1,35 @@ +#include "snmalloc.h" + +#include +#include + +int main(int argc, char* argv[]) +{ + if (argc != 2) + { + std::cerr << "Call with output file name" << std::endl; + return 1; + } + + // open a file in write mode. + ofstream outfile; + outfile.open(argv[1]); + + for (size_t size = 1024; size > 0; size -= 16) + { + auto sizeclass = snmalloc::size_to_sizeclass(size); + auto rsize = snmalloc::sizeclass_to_size(sizeclass); + if (rsize == size) + { + outfile << "DEFINE_MALLOC_SIZE(malloc_size_" << size << ", " << size + << ");" << std::endl; + } + else + { + outfile << "REDIRECT_MALLOC_SIZE(malloc_size_" << size << ", malloc_size_" + << rsize << ");" << std::endl; + } + } + + outfile.close(); +} \ No newline at end of file diff --git a/src/redirect/redirect.cc b/src/redirect/redirect.cc new file mode 100644 index 000000000..91526f330 --- /dev/null +++ b/src/redirect/redirect.cc @@ -0,0 +1,23 @@ + +#include "snmalloc.h" + +#define NAME(a) malloc_size_##a +#define STRINGIFY(a) a +#define NAME_STRING(a) NAME(a) + +#ifdef WIN32 +# define REDIRECT_MALLOC_SIZE(a, b) \ + extern "C" void* NAME(a)(); \ + __pragma(comment(linker, "/alternatename:malloc_size_##a=malloc_size_##b")) +#else +# define REDIRECT_MALLOC_SIZE(a, b) \ + __attribute__((alias(#b))) extern "C" void* a() +#endif + +#define DEFINE_MALLOC_SIZE(a, s) \ + extern "C" void* a() \ + { \ + return snmalloc::ThreadAlloc::get_noncachable()->template alloc(); \ + } + +#include "generated.cc" From 57dfeb1ac836e6f58cae6b9418bb836b10993324 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 5 Feb 2021 20:18:42 +0000 Subject: [PATCH 4/8] Typo --- src/mem/alloc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 9faae0f05..666eb4e6c 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -1071,7 +1071,7 @@ namespace snmalloc } template - SNMALLOC_FAST_PATH void* + SNMALLOC_SLOW_PATH void* small_alloc_inner_slow(sizeclass_t sizeclass, size_t size) { if (likely(!has_messages())) From 28c9d091fb59e61c45e99c5d51616bc2bf7aa1c9 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 11 Feb 2021 10:14:08 +0000 Subject: [PATCH 5/8] Add alignment --- CMakeLists.txt | 2 +- src/redirect/generate.cc | 26 +++++++++++++++----------- src/redirect/redirect.cc | 4 ++-- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ee809b413..16c0a3531 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,7 +180,7 @@ add_custom_target(generated ALL add_library(redirect_small STATIC src/redirect/redirect.cc) target_link_libraries(redirect_small snmalloc_lib) target_include_directories(redirect_small PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) - +add_dependencies(redirect_small generated) # To build with just the header library target define SNMALLOC_ONLY_HEADER_LIBRARY # in containing Cmake file. diff --git a/src/redirect/generate.cc b/src/redirect/generate.cc index e190aee07..d01aba381 100644 --- a/src/redirect/generate.cc +++ b/src/redirect/generate.cc @@ -15,19 +15,23 @@ int main(int argc, char* argv[]) ofstream outfile; outfile.open(argv[1]); - for (size_t size = 1024; size > 0; size -= 16) + for (size_t align = 0; align < 10; align++) { - auto sizeclass = snmalloc::size_to_sizeclass(size); - auto rsize = snmalloc::sizeclass_to_size(sizeclass); - if (rsize == size) + for (size_t size = 1024; size > 0; size -= 16) { - outfile << "DEFINE_MALLOC_SIZE(malloc_size_" << size << ", " << size - << ");" << std::endl; - } - else - { - outfile << "REDIRECT_MALLOC_SIZE(malloc_size_" << size << ", malloc_size_" - << rsize << ");" << std::endl; + auto asize = snmalloc::aligned_size(1ULL << align, size); + auto sizeclass = snmalloc::size_to_sizeclass(asize); + auto rsize = snmalloc::sizeclass_to_size(sizeclass); + if (rsize == size && align == 0) + { + outfile << "DEFINE_MALLOC_SIZE(malloc_size_" << size << "_" << align << ", " << size + << ");" << std::endl; + } + else + { + outfile << "REDIRECT_MALLOC_SIZE(malloc_size_" << size << "_" << align << ", malloc_size_" + << rsize << "_" << 0 << ");" << std::endl; + } } } diff --git a/src/redirect/redirect.cc b/src/redirect/redirect.cc index 91526f330..1fed48745 100644 --- a/src/redirect/redirect.cc +++ b/src/redirect/redirect.cc @@ -14,8 +14,8 @@ __attribute__((alias(#b))) extern "C" void* a() #endif -#define DEFINE_MALLOC_SIZE(a, s) \ - extern "C" void* a() \ +#define DEFINE_MALLOC_SIZE(name, s) \ + extern "C" void* name() \ { \ return snmalloc::ThreadAlloc::get_noncachable()->template alloc(); \ } From d48fd1ec691aadc612c8d1e21bcf1f88465a21a9 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 11 Feb 2021 10:24:07 +0000 Subject: [PATCH 6/8] Change API --- src/redirect/generate.cc | 6 ++++-- src/redirect/redirect.cc | 10 ++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/redirect/generate.cc b/src/redirect/generate.cc index d01aba381..cccbf177a 100644 --- a/src/redirect/generate.cc +++ b/src/redirect/generate.cc @@ -24,14 +24,16 @@ int main(int argc, char* argv[]) auto rsize = snmalloc::sizeclass_to_size(sizeclass); if (rsize == size && align == 0) { - outfile << "DEFINE_MALLOC_SIZE(malloc_size_" << size << "_" << align << ", " << size + outfile << "DEFINE_MALLOC_SIZE(__stack_alloc_small_" << size << "_" << align << ", " << size << ");" << std::endl; } else { - outfile << "REDIRECT_MALLOC_SIZE(malloc_size_" << size << "_" << align << ", malloc_size_" + outfile << "REDIRECT_MALLOC_SIZE(__stack_alloc_small_" << size << "_" << align << ", __stack_alloc_small_" << rsize << "_" << 0 << ");" << std::endl; } + outfile << "GENERATE_FREE_SIZE(__stack_free_small_" << size << "_" << align << ");" << std::endl; + } } diff --git a/src/redirect/redirect.cc b/src/redirect/redirect.cc index 1fed48745..0888db794 100644 --- a/src/redirect/redirect.cc +++ b/src/redirect/redirect.cc @@ -20,4 +20,14 @@ return snmalloc::ThreadAlloc::get_noncachable()->template alloc(); \ } +extern "C" void free_local_small(void* ptr) +{ + if (snmalloc::Alloc::small_local_dealloc(ptr)) + return; + snmalloc::ThreadAlloc::get_noncachable()->small_local_dealloc_slow(ptr); +} + +# define GENERATE_FREE_SIZE(a) \ + __attribute__((alias("free_local_small"))) extern "C" void* a() + #include "generated.cc" From f132034127486acdeb1946a8573f938c68ae3ef5 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 11 Feb 2021 10:37:04 +0000 Subject: [PATCH 7/8] Large API --- src/redirect/redirect.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/redirect/redirect.cc b/src/redirect/redirect.cc index 0888db794..bb7e58d31 100644 --- a/src/redirect/redirect.cc +++ b/src/redirect/redirect.cc @@ -30,4 +30,17 @@ extern "C" void free_local_small(void* ptr) # define GENERATE_FREE_SIZE(a) \ __attribute__((alias("free_local_small"))) extern "C" void* a() +void* __stack_alloc_large(size_t size, size_t align) +{ + size_t asize = snmalloc::aligned_size(1ULL << align, size); + return snmalloc::ThreadAlloc::get_noncachable()->alloc(asize); +} + +void __stack_free_large(void* ptr, size_t size, size_t align) +{ + size_t asize = snmalloc::aligned_size(1ULL << align, size); + snmalloc::ThreadAlloc::get_noncachable()->dealloc(ptr, asize); +} + + #include "generated.cc" From d2166fc820e3f42cd528c5be192c22147e71acc1 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 11 Feb 2021 10:45:32 +0000 Subject: [PATCH 8/8] Fix Windows Build Fix Align to be not a log size. --- src/redirect/redirect.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/redirect/redirect.cc b/src/redirect/redirect.cc index bb7e58d31..aadeeba56 100644 --- a/src/redirect/redirect.cc +++ b/src/redirect/redirect.cc @@ -8,7 +8,7 @@ #ifdef WIN32 # define REDIRECT_MALLOC_SIZE(a, b) \ extern "C" void* NAME(a)(); \ - __pragma(comment(linker, "/alternatename:malloc_size_##a=malloc_size_##b")) + __pragma(comment(linker, "/alternatename:##a=##b")) #else # define REDIRECT_MALLOC_SIZE(a, b) \ __attribute__((alias(#b))) extern "C" void* a() @@ -27,20 +27,25 @@ extern "C" void free_local_small(void* ptr) snmalloc::ThreadAlloc::get_noncachable()->small_local_dealloc_slow(ptr); } +#ifdef WIN32 +# define GENERATE_FREE_SIZE(a) \ + extern "C" void* NAME(a)(); \ + __pragma(comment(linker, "/alternatename:##a=free_local_small")) +#else # define GENERATE_FREE_SIZE(a) \ __attribute__((alias("free_local_small"))) extern "C" void* a() +#endif void* __stack_alloc_large(size_t size, size_t align) { - size_t asize = snmalloc::aligned_size(1ULL << align, size); + size_t asize = snmalloc::aligned_size(align, size); return snmalloc::ThreadAlloc::get_noncachable()->alloc(asize); } void __stack_free_large(void* ptr, size_t size, size_t align) { - size_t asize = snmalloc::aligned_size(1ULL << align, size); + size_t asize = snmalloc::aligned_size(align, size); snmalloc::ThreadAlloc::get_noncachable()->dealloc(ptr, asize); } - #include "generated.cc"