diff --git a/.github/workflows/dax.yml b/.github/workflows/dax.yml index 889825eaa..d3455df79 100644 --- a/.github/workflows/dax.yml +++ b/.github/workflows/dax.yml @@ -11,7 +11,7 @@ # # The FSDAX device should be mounted in the OS (e.g. /mnt/pmem1) # and the UMF_TESTS_FSDAX_PATH environment variable -# should contain a path to a file o this FSDAX device. +# should contain a path to a file on this FSDAX device. # name: Dax @@ -96,6 +96,6 @@ jobs: - name: Run the FSDAX tests working-directory: ${{env.BUILD_DIR}} - run: > - UMF_TESTS_FSDAX_PATH=${{env.UMF_TESTS_FSDAX_PATH}} - ctest -C ${{matrix.build_type}} -R umf-provider_file_memory -V + run: | + UMF_TESTS_FSDAX_PATH=${{env.UMF_TESTS_FSDAX_PATH}} ctest -C ${{matrix.build_type}} -R umf-provider_file_memory -V + UMF_TESTS_FSDAX_PATH=${{env.UMF_TESTS_FSDAX_PATH}} ctest -C ${{matrix.build_type}} -R umf_example_dram_and_fsdax -V diff --git a/CMakeLists.txt b/CMakeLists.txt index d926b7ec8..3ce5028c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -355,9 +355,12 @@ if(UMF_BUILD_LIBUMF_POOL_JEMALLOC) if(NOT JEMALLOC_FOUND) find_package(JEMALLOC REQUIRED jemalloc) endif() - # add PATH to DLL on Windows - set(DLL_PATH_LIST - "${DLL_PATH_LIST};PATH=path_list_append:${JEMALLOC_DLL_DIRS}") + if(JEMALLOC_FOUND OR JEMALLOC_LIBRARIES) + set(UMF_POOL_JEMALLOC_ENABLED TRUE) + # add PATH to DLL on Windows + set(DLL_PATH_LIST + "${DLL_PATH_LIST};PATH=path_list_append:${JEMALLOC_DLL_DIRS}") + endif() endif() if(WINDOWS) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 763d11670..99a1a58b0 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -269,6 +269,19 @@ if(LINUX) COMMAND ${EXAMPLE_NAME} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + if(UMF_POOL_JEMALLOC_ENABLED) + set(EXAMPLE_NAME umf_example_dram_and_fsdax) + + add_umf_executable( + NAME ${EXAMPLE_NAME} + SRCS dram_and_fsdax/dram_and_fsdax.c + LIBS umf jemalloc_pool) + + add_test( + NAME ${EXAMPLE_NAME} + COMMAND ${EXAMPLE_NAME} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + endif() else() message( STATUS "Memspace examples API are supported on Linux only - skipping") diff --git a/examples/cmake/FindJEMALLOC.cmake b/examples/cmake/FindJEMALLOC.cmake new file mode 100644 index 000000000..89d488ecc --- /dev/null +++ b/examples/cmake/FindJEMALLOC.cmake @@ -0,0 +1,52 @@ +# Copyright (C) 2024 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +message(STATUS "Checking for module 'jemalloc' using find_library()") + +find_library(JEMALLOC_LIBRARY NAMES libjemalloc jemalloc) +set(JEMALLOC_LIBRARIES ${JEMALLOC_LIBRARY}) + +get_filename_component(JEMALLOC_LIB_DIR ${JEMALLOC_LIBRARIES} DIRECTORY) +set(JEMALLOC_LIBRARY_DIRS ${JEMALLOC_LIB_DIR}) + +find_file(JEMALLOC_HEADER NAMES "jemalloc/jemalloc.h") +if(JEMALLOC_HEADER) + get_filename_component(JEMALLOC_INCLUDE_DIR_TBB ${JEMALLOC_HEADER} + DIRECTORY) + get_filename_component(JEMALLOC_INCLUDE_DIR ${JEMALLOC_INCLUDE_DIR_TBB} + DIRECTORY) + set(JEMALLOC_INCLUDE_DIRS ${JEMALLOC_INCLUDE_DIR}) +else() + set(MSG_NOT_FOUND " header NOT found " + "(set CMAKE_PREFIX_PATH to point the location)") + if(JEMALLOC_FIND_REQUIRED) + message(FATAL_ERROR ${MSG_NOT_FOUND}) + else() + message(WARNING ${MSG_NOT_FOUND}) + endif() +endif() + +if(WINDOWS) + find_file(JEMALLOC_DLL NAMES "bin/jemalloc.dll" "jemalloc.dll") + get_filename_component(JEMALLOC_DLL_DIR ${JEMALLOC_DLL} DIRECTORY) + set(JEMALLOC_DLL_DIRS ${JEMALLOC_DLL_DIR}) +endif() + +if(JEMALLOC_LIBRARY) + message(STATUS " Found jemalloc using find_library()") + message(STATUS " JEMALLOC_LIBRARIES = ${JEMALLOC_LIBRARIES}") + message(STATUS " JEMALLOC_INCLUDE_DIRS = ${JEMALLOC_INCLUDE_DIRS}") + message(STATUS " JEMALLOC_LIBRARY_DIRS = ${JEMALLOC_LIBRARY_DIRS}") + if(WINDOWS) + message(STATUS " JEMALLOC_DLL_DIRS = ${JEMALLOC_DLL_DIRS}") + endif() +else() + set(MSG_NOT_FOUND + "jemalloc NOT found (set CMAKE_PREFIX_PATH to point the location)") + if(JEMALLOC_FIND_REQUIRED) + message(FATAL_ERROR ${MSG_NOT_FOUND}) + else() + message(WARNING ${MSG_NOT_FOUND}) + endif() +endif() diff --git a/examples/dram_and_fsdax/CMakeLists.txt b/examples/dram_and_fsdax/CMakeLists.txt new file mode 100644 index 000000000..0d0bf2593 --- /dev/null +++ b/examples/dram_and_fsdax/CMakeLists.txt @@ -0,0 +1,61 @@ +# Copyright (C) 2024 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14.0 FATAL_ERROR) +project(umf_example_dram_and_fsdax LANGUAGES C) +enable_testing() + +set(UMF_EXAMPLE_DIR "${CMAKE_SOURCE_DIR}/..") +list(APPEND CMAKE_MODULE_PATH "${UMF_EXAMPLE_DIR}/cmake") +message(STATUS "CMAKE_MODULE_PATH=${CMAKE_MODULE_PATH}") + +find_package(PkgConfig) +pkg_check_modules(LIBUMF libumf) +if(NOT LIBUMF_FOUND) + find_package(LIBUMF REQUIRED libumf) +endif() + +pkg_check_modules(LIBHWLOC hwloc>=2.3.0) +if(NOT LIBHWLOC_FOUND) + find_package(LIBHWLOC 2.3.0 REQUIRED hwloc) +endif() + +pkg_check_modules(JEMALLOC jemalloc) +if(NOT JEMALLOC_FOUND) + find_package(JEMALLOC REQUIRED jemalloc) +endif() + +# build the example +set(EXAMPLE_NAME umf_example_dram_and_fsdax) +add_executable(${EXAMPLE_NAME} dram_and_fsdax.c) +target_include_directories(${EXAMPLE_NAME} PRIVATE ${LIBUMF_INCLUDE_DIRS}) + +target_link_directories( + ${EXAMPLE_NAME} + PRIVATE + ${LIBUMF_LIBRARY_DIRS} + ${LIBHWLOC_LIBRARY_DIRS} + ${JEMALLOC_LIBRARY_DIRS}) + +target_link_libraries( + ${EXAMPLE_NAME} PRIVATE hwloc jemalloc_pool ${JEMALLOC_LIBRARIES} + ${LIBUMF_LIBRARIES}) + +# an optional part - adds a test of this example +add_test( + NAME ${EXAMPLE_NAME} + COMMAND ${EXAMPLE_NAME} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + +set_tests_properties(${EXAMPLE_NAME} PROPERTIES LABELS "example-standalone") + +if(LINUX) + # set LD_LIBRARY_PATH + set_property( + TEST ${EXAMPLE_NAME} + PROPERTY + ENVIRONMENT_MODIFICATION + "LD_LIBRARY_PATH=path_list_append:${LIBUMF_LIBRARY_DIRS};LD_LIBRARY_PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS};LD_LIBRARY_PATH=path_list_append:${JEMALLOC_LIBRARY_DIRS}" + ) +endif() diff --git a/examples/dram_and_fsdax/dram_and_fsdax.c b/examples/dram_and_fsdax/dram_and_fsdax.c new file mode 100644 index 000000000..bc985692f --- /dev/null +++ b/examples/dram_and_fsdax/dram_and_fsdax.c @@ -0,0 +1,164 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include + +#include +#include +#include + +#include +#include + +static umf_memory_pool_handle_t create_dram_pool(void) { + umf_memory_provider_handle_t provider_dram = NULL; + umf_memory_pool_handle_t pool_dram; + umf_result_t umf_result; + + umf_os_memory_provider_params_t params_dram = + umfOsMemoryProviderParamsDefault(); + + umf_result = umfMemoryProviderCreate(umfOsMemoryProviderOps(), ¶ms_dram, + &provider_dram); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, "Creation of the OS memory provider failed"); + return NULL; + } + + // Create a DRAM memory pool + umf_result = umfPoolCreate(umfJemallocPoolOps(), provider_dram, NULL, + UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &pool_dram); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, "Failed to create a DRAM memory pool!\n"); + umfMemoryProviderDestroy(provider_dram); + return NULL; + } + + return pool_dram; +} + +static umf_memory_pool_handle_t create_fsdax_pool(const char *path) { + umf_memory_provider_handle_t provider_fsdax = NULL; + umf_memory_pool_handle_t pool_fsdax; + umf_result_t umf_result; + + umf_file_memory_provider_params_t params_fsdax = + umfFileMemoryProviderParamsDefault(path); + // FSDAX requires mapping the UMF_MEM_MAP_SYNC flag + params_fsdax.visibility = UMF_MEM_MAP_SYNC; + + umf_result = umfMemoryProviderCreate(umfFileMemoryProviderOps(), + ¶ms_fsdax, &provider_fsdax); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, "Failed to create the FSDAX file provider"); + return NULL; + } + + // Create an FSDAX memory pool + // + // The file memory provider does not support the free operation + // (`umfMemoryProviderFree()` always returns `UMF_RESULT_ERROR_NOT_SUPPORTED`), + // so it should be used with a pool manager that will take over + // the managing of the provided memory - for example the jemalloc pool + // with the `disable_provider_free` parameter set to true. + umf_jemalloc_pool_params_t pool_params; + pool_params.disable_provider_free = true; + + // Create an FSDAX memory pool + umf_result = + umfPoolCreate(umfJemallocPoolOps(), provider_fsdax, &pool_params, + UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &pool_fsdax); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, "Failed to create an FSDAX memory pool!\n"); + umfMemoryProviderDestroy(provider_fsdax); + return NULL; + } + + return pool_fsdax; +} + +int main(void) { + int ret = -1; + + // This example requires: + // - the FSDAX device to be mounted in the OS (e.g. /mnt/pmem1) and + // - the UMF_TESTS_FSDAX_PATH environment variable to contain + // a path to a file on this FSDAX device. + char *path = getenv("UMF_TESTS_FSDAX_PATH"); + if (path == NULL || path[0] == 0) { + fprintf( + stderr, + "Warning: UMF_TESTS_FSDAX_PATH is not set, skipping testing ...\n"); + return 0; + } + + umf_memory_pool_handle_t dram_pool = create_dram_pool(); + if (dram_pool == NULL) { + fprintf(stderr, "Failed to create a DRAM memory pool!\n"); + return -1; + } + + fprintf(stderr, "Created a DRAM memory pool\n"); + + umf_memory_pool_handle_t fsdax_pool = create_fsdax_pool(path); + if (fsdax_pool == NULL) { + fprintf(stderr, "Failed to create an FSDAX memory pool!\n"); + goto err_destroy_dram_pool; + } + + fprintf(stderr, "Created an FSDAX memory pool\n"); + + size_t size = 2 * 1024 * 1024; // == 2 MB + + // Allocate from the DRAM memory pool + char *dram_buf = umfPoolCalloc(dram_pool, 1, size); + if (dram_buf == NULL) { + fprintf(stderr, + "Failed to allocate memory from the DRAM memory pool!\n"); + goto err_destroy_pools; + } + + fprintf(stderr, "Allocated memory from the DRAM memory pool\n"); + + // Allocate from the FSDAX memory pool + char *fsdax_buf = umfPoolCalloc(fsdax_pool, 1, size); + if (fsdax_buf == NULL) { + fprintf(stderr, + "Failed to allocate memory from the FSDAX memory pool!\n"); + goto err_free_dram; + } + + fprintf(stderr, "Allocated memory from the FSDAX memory pool\n"); + + // Use the allocation from DRAM + dram_buf[0] = '.'; + + // Use the allocation from FSDAX + fsdax_buf[0] = '.'; + + // success + ret = 0; + + // The file memory provider does not support the free() operation, + // so we do not need to call: umfPoolFree(fsdax_pool, fsdax_buf); + +err_free_dram: + fprintf(stderr, "Freeing the allocation from the DRAM memory pool ...\n"); + umfPoolFree(dram_pool, dram_buf); + +err_destroy_pools: + fprintf(stderr, "Destroying the FSDAX memory pool ...\n"); + umfPoolDestroy(fsdax_pool); + +err_destroy_dram_pool: + fprintf(stderr, "Destroying the DRAM memory pool ...\n"); + umfPoolDestroy(dram_pool); + + return ret; +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6bbd7c54a..96640cb1f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -541,7 +541,7 @@ if(LINUX ) endif() - if(LINUX AND UMF_POOL_SCALABLE_ENABLED) + if(UMF_POOL_SCALABLE_ENABLED) set(EXAMPLES ${EXAMPLES} ipc_ipcapi) else() message( @@ -550,14 +550,27 @@ if(LINUX ) endif() + if(UMF_POOL_JEMALLOC_ENABLED) + set(EXAMPLES ${EXAMPLES} dram_and_fsdax) + else() + message( + STATUS + "The dram_and_fsdax example is supported on Linux only and requires UMF_BUILD_LIBUMF_POOL_JEMALLOC to be turned ON - skipping" + ) + endif() + if(EXAMPLES AND NOT UMF_DISABLE_HWLOC) + set(STANDALONE_CMAKE_OPTIONS + "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" + ) add_test( NAME umf-standalone_examples COMMAND ${UMF_CMAKE_SOURCE_DIR}/test/test_examples.sh ${UMF_CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/umf_standalone_examples/install-dir - "${CMAKE_INSTALL_PREFIX}" ${EXAMPLES} + "${CMAKE_INSTALL_PREFIX}" "${STANDALONE_CMAKE_OPTIONS}" + ${EXAMPLES} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) endif() endif() diff --git a/test/test_examples.sh b/test/test_examples.sh index 1d7e93ee1..efc86bcf9 100755 --- a/test/test_examples.sh +++ b/test/test_examples.sh @@ -9,15 +9,16 @@ SOURCE_DIR=$1 BUILD_DIR=$2 INSTALL_DIR=$3 CMAKE_INSTALL_PREFIX=$4 +STANDALONE_CMAKE_OPTIONS=$5 echo "Running: $0 $*" function print_usage() { echo "$(basename $0) - test all examples standalone" - echo "Usage: $(basename $0) " + echo "Usage: $(basename $0) " } -if [ "$5" = "" ]; then +if [ "$6" = "" ]; then print_usage echo -e "Error: too few arguments\n" exit 1 @@ -39,8 +40,9 @@ echo "SOURCE_DIR=$SOURCE_DIR" echo "BUILD_DIR=$BUILD_DIR" echo "CMAKE_INSTALL_PREFIX=$CMAKE_INSTALL_PREFIX" echo "INSTALL_DIR=$INSTALL_DIR" +echo "STANDALONE_CMAKE_OPTIONS=$STANDALONE_CMAKE_OPTIONS" -shift 4 +shift 5 EXAMPLES="$*" echo "Examples to run: $EXAMPLES" echo @@ -70,7 +72,7 @@ for ex in $EXAMPLES; do rm -rf $BLD_DIR mkdir -p $BLD_DIR cd $BLD_DIR - CMAKE_PREFIX_PATH="${INSTALL_DIR}/${CMAKE_INSTALL_PREFIX}" cmake $SRC_DIR + CMAKE_PREFIX_PATH="${INSTALL_DIR}/${CMAKE_INSTALL_PREFIX}" cmake $SRC_DIR $STANDALONE_CMAKE_OPTIONS make -j$(nproc) ctest --output-on-failure set +x