Sdallocx static jemalloc

.circleci/config.yml

@@ -29,7 +29,7 @@ jobs:
       - run:
           name: Installing dependencies (common)
           command: |
-            sudo apt-get install -y libboost-dev
+            sudo apt-get install -y libboost-dev libjemalloc-dev
       - when:
           condition:
             equal: [ "gcc", << parameters.compiler >> ]

.github/workflows/build.yml

@@ -225,7 +225,7 @@ jobs:
 
       - name: Setup common dependencies for Linux
         run: |
-          sudo apt-get install -y libboost-dev libc6-dev-i386
+          sudo apt-get install -y libboost-dev libc6-dev-i386 libjemalloc-dev
           sudo apt-get remove clang-tidy-11
         if: runner.os == 'Linux'
 
@@ -268,7 +268,7 @@ jobs:
 
       - name: Set up dependencies for macOS (common)
         run: |
-          brew install boost
+          brew install boost jemalloc
         if: runner.os == 'macOS'
 
       - name: Set up dependencies for macOS (coverage)

.github/workflows/experimental-build.yml

@@ -28,7 +28,7 @@ jobs:
       - name: Set up dependencies for macOS
         run: |
           brew update
-          brew install boost cppcheck iwyu
+          brew install boost cppcheck iwyu jemalloc
 
       - name: Create build environment
         run: mkdir ${{github.workspace}}/build

.github/workflows/msvc-build.yml

@@ -53,11 +53,17 @@ jobs:
           platform_version: 2019
           toolset: msvc
 
+      - name: Install jemalloc
+        run: |
+          vcpkg install jemalloc
+
       - name: Setup command line tools
         uses: ilammy/msvc-dev-cmd@v1
 
       - name: Configure CMake
-        run: cmake --preset "${{ matrix.preset }}"
+        run: |
+          $env:PKG_CONFIG_PATH += ";C:\vcpkg\packages\jemalloc_x86-windows\lib\pkgconfig"
+          cmake --preset "${{ matrix.preset }}"
         env:
           BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }}
 

.github/workflows/old-compilers.yml

@@ -355,7 +355,7 @@ jobs:
 
       - name: Setup common dependencies for Linux
         run: |
-          sudo apt-get install -y libboost-dev libc6-dev-i386
+          sudo apt-get install -y libboost-dev libc6-dev-i386 libjemalloc-dev
         if: runner.os == 'Linux'
 
       - name: Setup dependencies for Linux LLVM 11 & 12

.github/workflows/sonarcloud.yml

@@ -37,7 +37,7 @@ jobs:
       - name: Setup dependencies
         run: |
           sudo add-apt-repository -y 'ppa:mhier/libboost-latest'
-          sudo apt-get install -y boost1.74
+          sudo apt-get install -y boost1.74 libjemalloc-dev
 
       - name: Produce Compilation Database
         shell: bash

CMakeLists.txt

@@ -256,6 +256,20 @@ find_package(Threads REQUIRED)
 
 find_package(Boost REQUIRED)
 
+if(NOT("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin"
+      AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "arm64"))
+  set(USE_JEMALLOC ON)
+else()
+  set(USE_JEMALLOC OFF)
+endif()
+
+if(USE_JEMALLOC)
+  find_package(PkgConfig REQUIRED)
+
+  pkg_check_modules(JEMALLOC jemalloc)
+  pkg_search_module(JEMALLOC REQUIRED jemalloc)
+endif()
+
 string(REPLACE ";" " " CXX_FLAGS_FOR_SUBDIR_STR "${SANITIZER_CXX_FLAGS}")
 if(MSVC)
   string(REPLACE ";" " " MSVC_WARNING_FLAGS_FOR_SUBDIR_STR_TMP "${MSVC_CXX_WARNING_FLAGS}")
@@ -395,6 +409,7 @@ set(is_clang_ge_13_not_windows "$<AND:${is_clang_not_windows},${cxx_ge_13}>")
 set(is_clang_ge_14_not_windows "$<AND:${is_clang_not_windows},${cxx_ge_14}>")
 set(is_gxx_ge_11 "$<AND:${is_gxx},${cxx_ge_11}>")
 set(is_gxx_ge_12 "$<AND:${is_gxx},${cxx_ge_12}>")
+set(has_jemalloc "$<BOOL:${USE_JEMALLOC}>")
 set(has_avx2 "$<BOOL:${AVX2}>")
 set(fatal_warnings_on "$<BOOL:${FATAL_WARNINGS}>")
 set(coverage_on "$<BOOL:${COVERAGE}>")
@@ -496,6 +511,9 @@ function(COMMON_TARGET_PROPERTIES TARGET)
   set_target_properties(${TARGET} PROPERTIES CXX_EXTENSIONS OFF)
   target_compile_definitions(${TARGET} PRIVATE
     "$<${is_standalone}:UNODB_DETAIL_STANDALONE>")
+  target_compile_definitions(${TARGET} PRIVATE
+    # FIXME(laurynas): UNODB_DETAIL_USE_SIZED_FREE
+    "$<${has_jemalloc}:UNODB_DETAIL_USE_JEMALLOC>")
   target_compile_options(${TARGET} PRIVATE
     "${CXX_FLAGS}" "${SANITIZER_CXX_FLAGS}"
     "$<${is_msvc}:${MSVC_CXX_FLAGS}>"
@@ -520,6 +538,8 @@ function(COMMON_TARGET_PROPERTIES TARGET)
     "$<$<AND:${is_release},${is_not_windows}>:$<IF:${coverage_on},-O0,-O3>>"
     # Misc
     "$<${coverage_on}:--coverage>")
+  target_link_libraries(${TARGET} PRIVATE
+    "$<${has_jemalloc}:${JEMALLOC_LDFLAGS}>")
   # Change to target_link_options on 3.13 minimum CMake version
   target_link_libraries(${TARGET} INTERFACE "$<${coverage_on}:--coverage>")
   target_link_libraries(${TARGET} PRIVATE

art_internal_impl.hpp

@@ -167,7 +167,11 @@ inline void basic_db_leaf_deleter<Header, Db>::operator()(
     leaf_type *to_delete) const noexcept {
   const auto leaf_size = to_delete->get_size();
 
+#ifdef UNODB_DETAIL_USE_JEMALLOC
+  free_sized_aligned(to_delete, leaf_size);
+#else
   free_aligned(to_delete);
+#endif
 
   db.decrement_leaf_count(leaf_size);
 }
@@ -177,7 +181,11 @@ inline void basic_db_inode_deleter<INode, Db>::operator()(
     INode *inode_ptr) noexcept {
   static_assert(std::is_trivially_destructible_v<INode>);
 
+#ifdef UNODB_DETAIL_USE_JEMALLOC
+  free_sized_aligned(inode_ptr, sizeof(INode));
+#else
   free_aligned(inode_ptr);
+#endif
 
   db.template decrement_inode_count<INode>();
 }

heap.hpp

@@ -17,6 +17,10 @@
 #include <malloc.h>
 #endif
 
+#ifdef UNODB_DETAIL_USE_JEMALLOC
+#include <jemalloc/jemalloc.h>
+#endif
+
 #include "assert.hpp"
 
 namespace unodb::test {
@@ -81,7 +85,7 @@ template <typename T>
 
   void* result;
 
-#ifndef _MSC_VER
+#if !defined(_MSC_VER) || defined(UNODB_DETAIL_USE_JEMALLOC)
   const auto err = posix_memalign(&result, alignment, size);
   if (UNODB_DETAIL_UNLIKELY(err != 0)) result = nullptr;
 #else
@@ -102,6 +106,7 @@ template <typename T>
   return result;
 }
 
+#ifndef UNODB_DETAIL_USE_JEMALLOC
 inline void free_aligned(void* ptr) noexcept {
 #ifndef _MSC_VER
   // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory,hicpp-no-malloc)
@@ -110,6 +115,29 @@ inline void free_aligned(void* ptr) noexcept {
   _aligned_free(ptr);
 #endif
 }
+#endif  // !UNODB_DETAIL_USE_JEMALLOC
+
+inline void free_sized(void* ptr, std::size_t size) noexcept {
+#ifdef UNODB_DETAIL_USE_JEMALLOC
+  sdallocx(ptr, size, 0);
+#else
+  (void)size;
+  // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory,hicpp-no-malloc)
+  free(ptr);
+#endif
+}
+
+inline void free_sized_aligned(void* ptr, std::size_t size) noexcept {
+#ifdef UNODB_DETAIL_USE_JEMALLOC
+  sdallocx(ptr, size, 0);
+#elif defined(_MSC_VER)
+  _aligned_free(ptr);
+#else
+  (void)size;
+  // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory,hicpp-no-malloc)
+  free(ptr);
+#endif
+}
 
 }  // namespace unodb::detail
 

qsbr.hpp

@@ -350,8 +350,12 @@ namespace detail {
 
 struct [[nodiscard]] deallocation_request final {
   void *const pointer;
+#ifdef UNODB_DETAIL_USE_JEMALLOC
+  std::size_t size;
+#endif
 
 #ifndef NDEBUG
+  // FIXME(laurynas): pass size too?
   using debug_callback = std::function<void(const void *)>;
 
   // Non-const to support move
@@ -360,19 +364,25 @@ struct [[nodiscard]] deallocation_request final {
 #endif
 
   explicit deallocation_request(void *pointer_
+#ifdef UNODB_DETAIL_USE_JEMALLOC
+                                ,
+                                std::size_t size_
+#endif
 #ifndef NDEBUG
                                 ,
                                 qsbr_epoch request_epoch_,
                                 debug_callback dealloc_callback_
 #endif
                                 ) noexcept
-      : pointer {
-    pointer_
-  }
+      : pointer{pointer_}
+#ifdef UNODB_DETAIL_USE_JEMALLOC
+        ,
+        size{size_}
+#endif
 #ifndef NDEBUG
-  , dealloc_callback{std::move(dealloc_callback_)}, request_epoch {
-    request_epoch_
-  }
+        ,
+        dealloc_callback{std::move(dealloc_callback_)},
+        request_epoch{request_epoch_}
 #endif
   {
 #ifndef NDEBUG
@@ -410,16 +420,15 @@ class [[nodiscard]] deferred_requests final {
       std::optional<bool> dealloc_epoch_single_thread_mode_
 #endif
       ) noexcept
-      : requests {
-    std::move(requests_)
-  }
+      : requests{std::move(requests_)}
 #ifndef NDEBUG
-  , orphaned_requests{orphaned_requests_}, dealloc_epoch{request_epoch_},
-      dealloc_epoch_single_thread_mode {
-    dealloc_epoch_single_thread_mode_
-  }
+        ,
+        orphaned_requests{orphaned_requests_},
+        dealloc_epoch{request_epoch_},
+        dealloc_epoch_single_thread_mode{dealloc_epoch_single_thread_mode_}
 #endif
-  {}
+  {
+  }
 
   deferred_requests(const deferred_requests &) noexcept = delete;
   deferred_requests(deferred_requests &&) noexcept = delete;
@@ -640,6 +649,10 @@ class qsbr final {
   UNODB_DETAIL_DISABLE_MSVC_WARNING(26447)
   static void deallocate(
       void *pointer
+#ifdef UNODB_DETAIL_USE_JEMALLOC
+      ,
+      std::size_t size
+#endif
 #ifndef NDEBUG
       ,
       const detail::deallocation_request::debug_callback &debug_callback
@@ -648,7 +661,11 @@ class qsbr final {
 #ifndef NDEBUG
     if (debug_callback != nullptr) debug_callback(pointer);
 #endif
+#ifdef UNODB_DETAIL_USE_JEMALLOC
+    detail::free_sized_aligned(pointer, size);
+#else
     detail::free_aligned(pointer);
+#endif
   }
   UNODB_DETAIL_RESTORE_MSVC_WARNINGS()
 
@@ -856,6 +873,10 @@ inline void qsbr_per_thread::on_next_epoch_deallocate(
   if (UNODB_DETAIL_UNLIKELY(single_thread_mode)) {
     advance_last_seen_epoch(single_thread_mode, current_global_epoch);
     qsbr::deallocate(pointer
+#ifdef UNODB_DETAIL_USE_JEMALLOC
+                     ,
+                     size
+#endif
 #ifndef NDEBUG
                      ,
                      dealloc_callback
@@ -867,6 +888,10 @@ inline void qsbr_per_thread::on_next_epoch_deallocate(
   if (last_seen_epoch != current_global_epoch) {
     detail::dealloc_request_vector new_current_requests;
     new_current_requests.emplace_back(pointer
+#ifdef UNODB_DETAIL_USE_JEMALLOC
+                                      ,
+                                      size
+#endif
 #ifndef NDEBUG
                                       ,
                                       current_global_epoch,
@@ -882,6 +907,10 @@ inline void qsbr_per_thread::on_next_epoch_deallocate(
   }
 
   current_interval_dealloc_requests.emplace_back(pointer
+#ifdef UNODB_DETAIL_USE_JEMALLOC
+                                                 ,
+                                                 size
+#endif
 #ifndef NDEBUG
                                                  ,
                                                  last_seen_epoch,
@@ -1008,6 +1037,10 @@ inline void deallocation_request::deallocate(
                         *dealloc_epoch == request_epoch.advance(2))));
 
   qsbr::deallocate(pointer
+#ifdef UNODB_DETAIL_USE_JEMALLOC
+                   ,
+                   size
+#endif
 #ifndef NDEBUG
                    ,
                    dealloc_callback

test_heap.cpp

@@ -35,9 +35,13 @@ void operator delete(void* ptr) noexcept {
 }
 
 UNODB_DETAIL_DISABLE_CLANG_WARNING("-Wmissing-prototypes")
-void operator delete(void* ptr, std::size_t) noexcept {
-  // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory,hicpp-no-malloc)
+void operator delete(void* ptr, std::size_t size) noexcept {
+#ifdef UNODB_DETAIL_USE_JEMALLOC
+  unodb::detail::free_sized(ptr, size);
+#else
+  (void)size;
   free(ptr);
+#endif
 }
 UNODB_DETAIL_RESTORE_CLANG_WARNINGS()