Link statically with custom jemalloc with disabled initial TLS

ldorau · ldorau · commit d3ec7b364a73 · 2024-11-27T12:11:36.000+01:00
Link statically with custom jemalloc built from sources
with the following options enabled:
--with-jemalloc-prefix=je_
--disable-initial-exec-tls

Signed-off-by: Lukasz Dorau &lt;lukasz.dorau@intel.com&gt;
diff --git a/.github/docker/ubuntu-20.04.Dockerfile b/.github/docker/ubuntu-20.04.Dockerfile
@@ -24,7 +24,6 @@ ARG BASE_DEPS="\
 
 # UMF's dependencies
 ARG UMF_DEPS="\
-	libjemalloc-dev \
 	libhwloc-dev \
 	libtbb-dev"
 
diff --git a/.github/docker/ubuntu-22.04.Dockerfile b/.github/docker/ubuntu-22.04.Dockerfile
@@ -24,7 +24,6 @@ ARG BASE_DEPS="\
 
 # UMF's dependencies
 ARG UMF_DEPS="\
-	libjemalloc-dev \
 	libhwloc-dev \
 	libtbb-dev"
 
diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml
@@ -31,7 +31,7 @@ jobs:
     - name: Install apt packages
       run: |
         sudo apt-get update
-        sudo apt-get install -y cmake hwloc libhwloc-dev libjemalloc-dev libnuma-dev libtbb-dev
+        sudo apt-get install -y cmake hwloc libhwloc-dev libnuma-dev libtbb-dev
 
     - name: Download Coverity
       run: |
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -67,7 +67,7 @@ jobs:
     - name: Install apt packages
       run: |
         sudo apt-get update
-        sudo apt-get install -y cmake hwloc libhwloc-dev libjemalloc-dev libnuma-dev libtbb-dev valgrind
+        sudo apt-get install -y cmake hwloc libhwloc-dev libnuma-dev libtbb-dev valgrind
 
     - name: Configure CMake
       run: >
diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml
@@ -18,6 +18,7 @@ jobs:
   ubuntu-build:
     name: Ubuntu
     strategy:
+      fail-fast: false
       matrix:
         os: ['ubuntu-20.04', 'ubuntu-22.04']
         build_type: [Debug, Release]
@@ -124,7 +125,7 @@ jobs:
     - name: Install apt packages
       run: |
         sudo apt-get update
-        sudo apt-get install -y clang cmake libnuma-dev libjemalloc-dev lcov
+        sudo apt-get install -y clang cmake libnuma-dev lcov
 
     - name: Install TBB apt package
       if: matrix.install_tbb == 'ON'
@@ -172,9 +173,10 @@ jobs:
         ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' && '-DUMF_USE_COVERAGE=ON' || '' }}
 
     - name: Build UMF
+      working-directory: ${{env.BUILD_DIR}}
       run: |
         ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh' || true }}
-        cmake --build ${{env.BUILD_DIR}} -j $(nproc)
+        make -j $(nproc) || make VERBOSE=1
 
     - name: Run tests
       working-directory: ${{env.BUILD_DIR}}
@@ -469,7 +471,7 @@ jobs:
         python3 -m pip install -r third_party/requirements.txt
 
     - name: Install hwloc
-      run: brew install hwloc jemalloc tbb
+      run: brew install hwloc tbb automake
 
     - name: Configure build
       run: >
diff --git a/.github/workflows/reusable_benchmarks.yml b/.github/workflows/reusable_benchmarks.yml
@@ -34,7 +34,7 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         run: |
           sudo apt-get update
-          sudo apt-get install -y cmake libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev
+          sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev
 
       - name: Initialize vcpkg
         if: matrix.os == 'windows-latest'
diff --git a/.github/workflows/reusable_codeql.yml b/.github/workflows/reusable_codeql.yml
@@ -62,7 +62,7 @@ jobs:
       if: matrix.os == 'ubuntu-latest'
       run: |
         sudo apt-get update
-        sudo apt-get install -y cmake clang libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev
+        sudo apt-get install -y cmake clang libhwloc-dev libnuma-dev libtbb-dev
 
     # Latest distros do not allow global pip installation
     - name: "[Lin] Install Python requirements in venv"
diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml
@@ -79,13 +79,13 @@ jobs:
       if: matrix.os == 'ubuntu-latest'
       run: |
         sudo apt-get update
-        sudo apt-get install -y cmake libjemalloc-dev libhwloc-dev libnuma-dev libtbb-dev
+        sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev
 
     - name: Install dependencies (ubuntu-20.04)
       if: matrix.os == 'ubuntu-20.04'
       run: |
         sudo apt-get update
-        sudo apt-get install -y cmake libjemalloc-dev libnuma-dev libtbb-dev
+        sudo apt-get install -y cmake libnuma-dev libtbb-dev
         .github/scripts/install_hwloc.sh # install hwloc-2.3.0 instead of hwloc-2.1.0 present in the OS package
 
     - name: Set ptrace value for IPC test (on Linux only)
diff --git a/.github/workflows/reusable_proxy_lib.yml b/.github/workflows/reusable_proxy_lib.yml
@@ -32,7 +32,7 @@ jobs:
       - name: Install apt packages
         run: |
           sudo apt-get update
-          sudo apt-get install -y cmake libhwloc-dev libjemalloc-dev libtbb-dev lcov
+          sudo apt-get install -y cmake libhwloc-dev libtbb-dev lcov
 
       - name: Set ptrace value for IPC test
         run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"
diff --git a/.github/workflows/reusable_sanitizers.yml b/.github/workflows/reusable_sanitizers.yml
@@ -29,7 +29,7 @@ jobs:
     - name: Install apt packages
       run: |
         sudo apt-get update
-        sudo apt-get install -y clang cmake libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev
+        sudo apt-get install -y clang cmake libhwloc-dev libnuma-dev libtbb-dev
 
     - name: Install oneAPI basekit
       if: matrix.compiler.cxx == 'icpx'
diff --git a/.github/workflows/reusable_valgrind.yml b/.github/workflows/reusable_valgrind.yml
@@ -20,7 +20,7 @@ jobs:
     - name: Install apt packages
       run: |
         sudo apt-get update
-        sudo apt-get install -y cmake hwloc libhwloc-dev libjemalloc-dev libnuma-dev libtbb-dev valgrind
+        sudo apt-get install -y cmake hwloc libhwloc-dev libnuma-dev libtbb-dev valgrind
 
     - name: Configure CMake
       run: >
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -111,6 +111,79 @@ else()
     message(FATAL_ERROR "Unknown OS type")
 endif()
 
+if(NOT UMF_BUILD_LIBUMF_POOL_JEMALLOC)
+    message(STATUS "jemalloc is disabled")
+elseif(WINDOWS)
+    pkg_check_modules(JEMALLOC jemalloc)
+    if(NOT JEMALLOC_FOUND)
+        find_package(JEMALLOC REQUIRED jemalloc)
+    endif()
+else()
+    if(NOT DEFINED UMF_JEMALLOC_REPO)
+        set(UMF_JEMALLOC_REPO "https://github.com/jemalloc/jemalloc.git")
+    endif()
+
+    if(NOT DEFINED UMF_JEMALLOC_TAG)
+        set(UMF_JEMALLOC_TAG 5.3.0)
+    endif()
+
+    include(FetchContent)
+    message(
+        STATUS
+            "Will fetch jemalloc from ${UMF_JEMALLOC_REPO} (tag: ${UMF_JEMALLOC_TAG})"
+    )
+
+    FetchContent_Declare(
+        jemalloc_targ
+        GIT_REPOSITORY ${UMF_JEMALLOC_REPO}
+        GIT_TAG ${UMF_JEMALLOC_TAG})
+    FetchContent_MakeAvailable(jemalloc_targ)
+
+    add_custom_command(
+        COMMAND ./autogen.sh
+        WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR}
+        OUTPUT ${jemalloc_targ_SOURCE_DIR}/configure)
+    add_custom_command(
+        COMMAND
+            ./configure --prefix=${jemalloc_targ_BINARY_DIR}
+            --with-jemalloc-prefix=je_ --disable-initial-exec-tls CFLAGS=-fPIC
+            CXXFLAGS=-fPIC
+        WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR}
+        OUTPUT ${jemalloc_targ_SOURCE_DIR}/Makefile
+        DEPENDS ${jemalloc_targ_SOURCE_DIR}/configure)
+    add_custom_command(
+        COMMAND make
+        WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR}
+        OUTPUT ${jemalloc_targ_SOURCE_DIR}/lib/libjemalloc.la
+        DEPENDS ${jemalloc_targ_SOURCE_DIR}/Makefile)
+    add_custom_command(
+        COMMAND make install
+        WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR}
+        OUTPUT ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a
+        DEPENDS ${jemalloc_targ_SOURCE_DIR}/lib/libjemalloc.la)
+
+    add_custom_target(jemalloc_prod
+                      DEPENDS ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a)
+    add_library(jemalloc INTERFACE)
+    target_link_libraries(
+        jemalloc INTERFACE ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a)
+    add_dependencies(jemalloc jemalloc_prod)
+
+    set(JEMALLOC_LIBRARY_DIRS ${jemalloc_targ_BINARY_DIR}/lib)
+    set(JEMALLOC_INCLUDE_DIRS ${jemalloc_targ_BINARY_DIR}/include)
+    set(JEMALLOC_LIBRARIES ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a)
+endif()
+
+if(JEMALLOC_FOUND OR JEMALLOC_LIBRARIES)
+    set(UMF_POOL_JEMALLOC_ENABLED TRUE)
+    # add PATH to DLL on Windows
+    set(DLL_PATH_LIST
+        "${DLL_PATH_LIST};PATH=path_list_append:${JEMALLOC_DLL_DIRS}")
+    message(STATUS "    JEMALLOC_LIBRARIES = ${JEMALLOC_LIBRARIES}")
+    message(STATUS "    JEMALLOC_INCLUDE_DIRS = ${JEMALLOC_INCLUDE_DIRS}")
+    message(STATUS "    JEMALLOC_LIBRARY_DIRS = ${JEMALLOC_LIBRARY_DIRS}")
+endif()
+
 if(UMF_DISABLE_HWLOC)
     message(STATUS "hwloc is disabled, hence OS provider, memtargets, "
                    "topology discovery, examples won't be available!")
@@ -402,19 +475,6 @@ else()
     set(UMF_POOL_SCALABLE_ENABLED FALSE)
 endif()
 
-if(UMF_BUILD_LIBUMF_POOL_JEMALLOC)
-    pkg_check_modules(JEMALLOC jemalloc)
-    if(NOT JEMALLOC_FOUND)
-        find_package(JEMALLOC REQUIRED jemalloc)
-    endif()
-    if(JEMALLOC_FOUND OR JEMALLOC_LIBRARIES)
-        set(UMF_POOL_JEMALLOC_ENABLED TRUE)
-        # add PATH to DLL on Windows
-        set(DLL_PATH_LIST
-            "${DLL_PATH_LIST};PATH=path_list_append:${JEMALLOC_DLL_DIRS}")
-    endif()
-endif()
-
 if(WINDOWS)
     # TODO: enable the proxy library in the Debug build on Windows
     #
diff --git a/README.md b/README.md
@@ -262,7 +262,7 @@ The `UMF_BUILD_LIBUMF_POOL_JEMALLOC` option has to be turned `ON` to build this
 
 1) The `UMF_BUILD_LIBUMF_POOL_JEMALLOC` option turned `ON`
 2) Required packages:
-   - libjemalloc-dev (Linux) or jemalloc (Windows)
+   - jemalloc (Windows only)
 
 #### Scalable Pool (part of libumf)
 
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
@@ -282,8 +282,11 @@ if(LINUX)
             SRCS dram_and_fsdax/dram_and_fsdax.c
             LIBS umf jemalloc_pool)
 
-        target_link_directories(${EXAMPLE_NAME} PRIVATE
-                                ${LIBHWLOC_LIBRARY_DIRS})
+        target_link_options(${EXAMPLE_NAME} PRIVATE "-Wl,--no-as-needed,-ldl")
+
+        target_link_directories(
+            ${EXAMPLE_NAME} PRIVATE ${LIBHWLOC_LIBRARY_DIRS}
+            ${JEMALLOC_LIBRARY_DIRS})
 
         add_test(
             NAME ${EXAMPLE_NAME}
diff --git a/examples/cmake/FindJEMALLOC.cmake b/examples/cmake/FindJEMALLOC.cmake
@@ -2,9 +2,11 @@
 # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-message(STATUS "Checking for module 'jemalloc' using find_library()")
+message(
+    STATUS "Looking for the static 'libjemalloc.a' library using find_library()"
+)
 
-find_library(JEMALLOC_LIBRARY NAMES libjemalloc jemalloc)
+find_library(JEMALLOC_LIBRARY NAMES libjemalloc.a jemalloc.a)
 set(JEMALLOC_LIBRARIES ${JEMALLOC_LIBRARY})
 
 get_filename_component(JEMALLOC_LIB_DIR ${JEMALLOC_LIBRARIES} DIRECTORY)
diff --git a/examples/dram_and_fsdax/CMakeLists.txt b/examples/dram_and_fsdax/CMakeLists.txt
@@ -21,10 +21,8 @@ if(NOT LIBHWLOC_FOUND)
     find_package(LIBHWLOC 2.3.0 REQUIRED hwloc)
 endif()
 
-pkg_check_modules(JEMALLOC jemalloc)
-if(NOT JEMALLOC_FOUND)
-    find_package(JEMALLOC REQUIRED jemalloc)
-endif()
+# find custom jemalloc pointed by CMAKE_PREFIX_PATH (with je_ prefix enabled)
+find_package(JEMALLOC REQUIRED jemalloc)
 
 # build the example
 set(EXAMPLE_NAME umf_example_dram_and_fsdax)
diff --git a/scripts/qemu/run-build.sh b/scripts/qemu/run-build.sh
@@ -14,7 +14,7 @@ pwd
 
 echo password | sudo -Sk apt-get update
 echo password | sudo -Sk apt-get install -y git cmake gcc g++ pkg-config \
-    numactl libnuma-dev hwloc libhwloc-dev libjemalloc-dev libtbb-dev valgrind lcov
+    numactl libnuma-dev hwloc libhwloc-dev libtbb-dev valgrind lcov
 
 mkdir build
 cd build
diff --git a/src/pool/CMakeLists.txt b/src/pool/CMakeLists.txt
@@ -45,10 +45,13 @@ if(UMF_BUILD_LIBUMF_POOL_JEMALLOC)
         NAME jemalloc_pool
         TYPE STATIC
         SRCS pool_jemalloc.c ${POOL_EXTRA_SRCS}
-        LIBS jemalloc ${POOL_EXTRA_LIBS})
+        LIBS ${JEMALLOC_LIBRARIES} ${POOL_EXTRA_LIBS})
     target_include_directories(jemalloc_pool PRIVATE ${JEMALLOC_INCLUDE_DIRS})
     target_compile_definitions(jemalloc_pool
                                PRIVATE ${POOL_COMPILE_DEFINITIONS})
     add_library(${PROJECT_NAME}::jemalloc_pool ALIAS jemalloc_pool)
+    if(NOT WINDOWS)
+        add_dependencies(jemalloc_pool jemalloc)
+    endif()
     install(TARGETS jemalloc_pool EXPORT ${PROJECT_NAME}-targets)
 endif()
diff --git a/src/pool/pool_jemalloc.c b/src/pool/pool_jemalloc.c
@@ -22,16 +22,6 @@
 
 #include <jemalloc/jemalloc.h>
 
-// The Windows version of jemalloc uses API with je_ prefix,
-// while the Linux one does not.
-#ifndef _WIN32
-#define je_mallocx mallocx
-#define je_dallocx dallocx
-#define je_rallocx rallocx
-#define je_mallctl mallctl
-#define je_malloc_usable_size malloc_usable_size
-#endif
-
 #define MALLOCX_ARENA_MAX (MALLCTL_ARENAS_ALL - 1)
 
 typedef struct jemalloc_memory_pool_t {
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -725,6 +725,13 @@ if(LINUX
         set(STANDALONE_CMAKE_OPTIONS
             "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
         )
+        if(JEMALLOC_INCLUDE_DIRS)
+            # add custom jemalloc installation
+            set(STANDALONE_CMAKE_OPTIONS
+                "${STANDALONE_CMAKE_OPTIONS} -DCMAKE_PREFIX_PATH=${JEMALLOC_INCLUDE_DIRS}/../"
+            )
+        endif()
+
         add_test(
             NAME umf-standalone_examples
             COMMAND
diff --git a/test/supp/drd-umf_test-jemalloc_coarse_devdax.supp b/test/supp/drd-umf_test-jemalloc_coarse_devdax.supp
@@ -1,27 +1,8 @@
 {
-   False-positive ConflictingAccess in libjemalloc.so
+   False-positive ConflictingAccess in jemalloc
    drd:ConflictingAccess
-   obj:*/libjemalloc.so*
    ...
-   fun:mallocx
-   ...
-}
-
-{
-   False-positive ConflictingAccess in libjemalloc.so
-   drd:ConflictingAccess
-   obj:*/libjemalloc.so*
-   ...
-   fun:op_free
-   ...
-}
-
-{
-   False-positive ConflictingAccess in libjemalloc.so
-   drd:ConflictingAccess
-   obj:*/libjemalloc.so*
-   ...
-   fun:__nptl_deallocate_tsd
+   fun:je_*
    ...
 }
 
diff --git a/test/supp/drd-umf_test-jemalloc_coarse_file.supp b/test/supp/drd-umf_test-jemalloc_coarse_file.supp
@@ -1,27 +1,8 @@
 {
-   False-positive ConflictingAccess in libjemalloc.so
+   False-positive ConflictingAccess in jemalloc
    drd:ConflictingAccess
-   obj:*/libjemalloc.so*
    ...
-   fun:mallocx
-   ...
-}
-
-{
-   False-positive ConflictingAccess in libjemalloc.so
-   drd:ConflictingAccess
-   obj:*/libjemalloc.so*
-   ...
-   fun:op_free
-   ...
-}
-
-{
-   False-positive ConflictingAccess in libjemalloc.so
-   drd:ConflictingAccess
-   obj:*/libjemalloc.so*
-   ...
-   fun:__nptl_deallocate_tsd
+   fun:je_*
    ...
 }
 
diff --git a/test/supp/drd-umf_test-jemalloc_pool.supp b/test/supp/drd-umf_test-jemalloc_pool.supp
diff --git a/test/supp/helgrind-umf_test-jemalloc_coarse_devdax.supp b/test/supp/helgrind-umf_test-jemalloc_coarse_devdax.supp
diff --git a/test/supp/helgrind-umf_test-jemalloc_coarse_file.supp b/test/supp/helgrind-umf_test-jemalloc_coarse_file.supp
diff --git a/test/supp/helgrind-umf_test-jemalloc_pool.supp b/test/supp/helgrind-umf_test-jemalloc_pool.supp
