diff --git a/.cmake-format b/.cmake-format index c1a8e85a83..f5f413d519 100644 --- a/.cmake-format +++ b/.cmake-format @@ -26,7 +26,8 @@ with section("parse"): 'kwargs': { 'NAME': '*', 'SRCS': '*', - 'LIBS': '*'}}, + 'LIBS': '*', + 'ENVS': '*'}}, 'add_umf_library': { "pargs": 0, "flags": [], diff --git a/.github/scripts/get_system_info.sh b/.github/scripts/get_system_info.sh index be900e2a73..81c54ce980 100755 --- a/.github/scripts/get_system_info.sh +++ b/.github/scripts/get_system_info.sh @@ -15,7 +15,7 @@ function check_L0_version { fi if command -v zypper &> /dev/null; then - zypper se level-zero && return + zypper -n se level-zero || true fi echo "level-zero not installed" diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 5a6756f2ca..41ce4b3850 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -74,6 +74,17 @@ jobs: install_tbb: 'ON' disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' + # test lld linker + - os: 'ubuntu-24.04' + build_type: Release + compiler: {c: icx, cxx: icpx} + shared_library: 'ON' + level_zero_provider: 'ON' + cuda_provider: 'ON' + install_tbb: 'ON' + disable_hwloc: 'OFF' + link_hwloc_statically: 'OFF' + llvm_linker: '-DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" -DCMAKE_MODULE_LINKER_FLAGS="-fuse-ld=lld" -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld"' # test without installing TBB - os: 'ubuntu-22.04' build_type: Release @@ -160,6 +171,7 @@ jobs: -DUMF_DISABLE_HWLOC=${{matrix.disable_hwloc}} -DUMF_LINK_HWLOC_STATICALLY=${{matrix.link_hwloc_statically}} ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' && '-DUMF_USE_COVERAGE=ON' || '' }} + ${{ matrix.llvm_linker || '' }} - name: Build UMF run: | diff --git a/.github/workflows/reusable_benchmarks.yml b/.github/workflows/reusable_benchmarks.yml index 15e6b15f4a..3953e98de5 100644 --- a/.github/workflows/reusable_benchmarks.yml +++ b/.github/workflows/reusable_benchmarks.yml @@ -103,15 +103,17 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: intel/llvm - # add preloaded UMF benchmarks - # https://github.com/intel/llvm/pull/17278 - ref: b2f9dab5266d227cc9eb19af1b54c5bdc50221d1 + # [BENCHMARK] fix default timeout parameter + # https://github.com/intel/llvm/pull/17412 + ref: 357e9e0b253b7eba105d044e38452b3c09169f8a path: sycl-repo fetch-depth: 1 - name: Install benchmarking scripts deps run: | - pip install --force-reinstall -r ${{github.workspace}}/sycl-repo/unified-runtime/third_party/benchmark_requirements.txt + python -m venv .venv + source .venv/bin/activate + pip install -r ${{github.workspace}}/sycl-repo/unified-runtime/third_party/benchmark_requirements.txt - name: Set core range and GPU mask run: | @@ -135,10 +137,12 @@ jobs: id: benchmarks working-directory: ${{env.BUILD_DIR}} run: > + source ${{github.workspace}}/.venv/bin/activate && taskset -c ${{ env.CORES }} ${{ github.workspace }}/sycl-repo/unified-runtime/scripts/benchmarks/main.py ~/bench_workdir_umf --umf ${{env.BUILD_DIR}} --compare baseline + --timeout 3000 ${{ inputs.upload_report && '--output-html' || '' }} ${{ inputs.pr_no != 0 && '--output-markdown' || '' }} ${{ inputs.bench_script_params }} diff --git a/.github/workflows/reusable_compatibility.yml b/.github/workflows/reusable_compatibility.yml index ec84cef763..5116a59f47 100644 --- a/.github/workflows/reusable_compatibility.yml +++ b/.github/workflows/reusable_compatibility.yml @@ -1,4 +1,4 @@ -# Workflow for checkig the backward compatibility of UMF. +# Workflow for checking the backward compatibility of UMF. # Test the latest UMF shared library with binaries compiled using the older UMF # shared library. name: Compatibility @@ -15,7 +15,7 @@ permissions: contents: read jobs: - ubuntu-build: + ubuntu: name: Ubuntu runs-on: 'ubuntu-22.04' @@ -35,12 +35,6 @@ jobs: - name: Install libhwloc working-directory: ${{github.workspace}}/tag_version run: .github/scripts/install_hwloc.sh - - - name: Get "tag" UMF version - working-directory: ${{github.workspace}}/tag_version - run: | - VERSION=$(git describe --tags) - echo "tag version: $VERSION" - name: Configure "tag" UMF build working-directory: ${{github.workspace}}/tag_version @@ -52,6 +46,7 @@ jobs: -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DUMF_BUILD_TESTS=ON + -DUMF_BUILD_EXAMPLES=ON -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON -DUMF_BUILD_CUDA_PROVIDER=ON -DUMF_FORMAT_CODE_STYLE=OFF @@ -64,7 +59,7 @@ jobs: working-directory: ${{github.workspace}}/tag_version run: | cmake --build ${{github.workspace}}/tag_version/build -j $(nproc) - + - name: Run "tag" UMF tests working-directory: ${{github.workspace}}/tag_version/build run: | @@ -75,13 +70,7 @@ jobs: with: fetch-depth: 0 path: ${{github.workspace}}/latest_version - - - name: Get latest UMF version - working-directory: ${{github.workspace}}/latest_version - run: | - VERSION=$(git describe --tags) - echo "checked version: $VERSION" - + - name: Configure latest UMF build working-directory: ${{github.workspace}}/latest_version run: > @@ -97,7 +86,6 @@ jobs: -DUMF_FORMAT_CODE_STYLE=OFF -DUMF_DEVELOPER_MODE=ON -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON - -DUMF_TESTS_FAIL_ON_SKIP=ON - name: Build latest UMF working-directory: ${{github.workspace}}/latest_version @@ -106,12 +94,14 @@ jobs: - name: Run "tag" UMF tests with latest UMF libs (warnings enabled) working-directory: ${{github.workspace}}/tag_version/build + # GTEST_FILTER is used below to skip test that is not compatible run: > - UMF_LOG="level:warning;flush:debug;output:stderr;pid:no" - LD_LIBRARY_PATH=${{github.workspace}}/latest_version/build/lib/ - ctest --output-on-failure -E "umf-mempolicy" # disable tests that rely on internal structures - - windows-build: + UMF_LOG="level:warning;flush:debug;output:stderr;pid:no" + LD_LIBRARY_PATH=${{github.workspace}}/latest_version/build/lib/ + GTEST_FILTER="-*umfIpcTest.GetPoolByOpenedHandle*" + ctest --verbose + + windows: name: Windows env: VCPKG_PATH: "${{github.workspace}}/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/vcpkg/packages/jemalloc_x64-windows" @@ -132,16 +122,11 @@ jobs: vcpkgDirectory: ${{github.workspace}}/vcpkg vcpkgJsonGlob: '**/vcpkg.json' + # NOTE we use vcpkg setup from "tag" version - name: Install dependencies working-directory: ${{github.workspace}}/tag_version run: vcpkg install shell: pwsh # Specifies PowerShell as the shell for running the script. - - - name: Get "tag" UMF version - working-directory: ${{github.workspace}}/tag_version - run: | - $version = (git describe --tags) - echo "tag version: $VERSION" - name: Configure "tag" UMF build working-directory: ${{github.workspace}}/tag_version @@ -153,6 +138,7 @@ jobs: -DCMAKE_CXX_COMPILER=cl -DUMF_BUILD_SHARED_LIBRARY=ON -DUMF_BUILD_TESTS=ON + -DUMF_BUILD_EXAMPLES=ON -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON -DUMF_BUILD_CUDA_PROVIDER=ON -DUMF_FORMAT_CODE_STYLE=OFF @@ -174,13 +160,6 @@ jobs: fetch-depth: 0 path: ${{github.workspace}}/latest_version - # NOTE we use vcpkg setup from "tag" version - - name: Get latest UMF version - working-directory: ${{github.workspace}}/latest_version - run: | - $version = (git describe --tags) - echo "latest version: $VERSION" - - name: Configure latest UMF build working-directory: ${{github.workspace}}/latest_version run: > @@ -196,7 +175,6 @@ jobs: -DUMF_FORMAT_CODE_STYLE=OFF -DUMF_DEVELOPER_MODE=ON -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON - -DUMF_TESTS_FAIL_ON_SKIP=ON - name: Build latest UMF run: cmake --build "${{github.workspace}}/latest_version/build" --config Debug -j $Env:NUMBER_OF_PROCESSORS @@ -205,5 +183,93 @@ jobs: working-directory: ${{github.workspace}}/tag_version/build run: | $env:UMF_LOG="level:warning;flush:debug;output:stderr;pid:no" + $env:GTEST_FILTER="-*umfIpcTest.GetPoolByOpenedHandle*" cp ${{github.workspace}}/latest_version/build/bin/Debug/umf.dll ${{github.workspace}}/tag_version/build/bin/Debug/umf.dll - ctest -C Debug --output-on-failure --test-dir test + ctest -C Debug --verbose + + gpu: + name: GPU Ubuntu + # run only on upstream; forks will not have the HW + if: github.repository == 'oneapi-src/unified-memory-framework' + strategy: + matrix: + provider: ['LEVEL_ZERO', 'CUDA'] + runs-on: ["DSS-${{matrix.provider}}", "DSS-UBUNTU"] + + steps: + - name: Checkout "tag" UMF version + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + ref: refs/tags/${{inputs.tag}} + path: ${{github.workspace}}/tag_version + + - name: Configure "tag" UMF build + working-directory: ${{github.workspace}}/tag_version + run: > + cmake + -B ${{github.workspace}}/tag_version/build + -DCMAKE_BUILD_TYPE=Debug + -DUMF_BUILD_SHARED_LIBRARY=ON + -DCMAKE_C_COMPILER=gcc + -DCMAKE_CXX_COMPILER=g++ + -DUMF_BUILD_TESTS=ON + -DUMF_BUILD_GPU_TESTS=ON + -DUMF_BUILD_EXAMPLES=ON + -DUMF_BUILD_GPU_EXAMPLES=ON + -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF + -DUMF_BUILD_CUDA_PROVIDER=OFF + -DUMF_BUILD_${{matrix.provider}}_PROVIDER=ON + -DUMF_FORMAT_CODE_STYLE=OFF + -DUMF_DEVELOPER_MODE=ON + -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON + -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON + -DUMF_TESTS_FAIL_ON_SKIP=ON + + - name: Build "tag" UMF + working-directory: ${{github.workspace}}/tag_version + run: | + cmake --build ${{github.workspace}}/tag_version/build -j $(nproc) + + - name: Run "tag" UMF tests + working-directory: ${{github.workspace}}/tag_version/build + run: > + LD_LIBRARY_PATH=${{github.workspace}}/tag_version/build/lib/ + GTEST_FILTER="-*umfIpcTest.GetPoolByOpenedHandle*" + ctest --output-on-failure + + - name: Checkout latest UMF version + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + path: ${{github.workspace}}/latest_version + + - name: Configure latest UMF build + working-directory: ${{github.workspace}}/latest_version + run: > + cmake + -B ${{github.workspace}}/latest_version/build + -DCMAKE_BUILD_TYPE=Debug + -DUMF_BUILD_SHARED_LIBRARY=ON + -DCMAKE_C_COMPILER=gcc + -DCMAKE_CXX_COMPILER=g++ + -DUMF_BUILD_TESTS=OFF + -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON + -DUMF_BUILD_CUDA_PROVIDER=ON + -DUMF_FORMAT_CODE_STYLE=OFF + -DUMF_DEVELOPER_MODE=ON + -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON + + - name: Build latest UMF + working-directory: ${{github.workspace}}/latest_version + run: | + cmake --build ${{github.workspace}}/latest_version/build -j $(nproc) + + # NOTE: Once not implemented features may now be implemented - exclude these tests + - name: Run "tag" UMF tests with latest UMF libs (warnings enabled) + working-directory: ${{github.workspace}}/tag_version/build + run: > + UMF_LOG="level:warning;flush:debug;output:stderr;pid:no" + LD_LIBRARY_PATH=${{github.workspace}}/latest_version/build/lib/ + GTEST_FILTER="-*umfIpcTest.GetPoolByOpenedHandle*" + ctest --verbose -E "not_impl" diff --git a/.github/workflows/reusable_gpu.yml b/.github/workflows/reusable_gpu.yml index a67be2f522..721d85206b 100644 --- a/.github/workflows/reusable_gpu.yml +++ b/.github/workflows/reusable_gpu.yml @@ -99,8 +99,8 @@ jobs: -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF -DUMF_BUILD_${{inputs.name}}_PROVIDER=ON -DUMF_TESTS_FAIL_ON_SKIP=ON - ${{ matrix.os == 'Ubuntu' && matrix.build_type == 'Debug' && '-DUMF_USE_COVERAGE=ON' || '' }} - ${{ matrix.os == 'Windows' && '-DCMAKE_SUPPRESS_REGENERATION=ON' || '' }} + ${{ matrix.os == 'Ubuntu' && matrix.build_type == 'Debug' && '-DUMF_USE_COVERAGE=ON' || '' }} + ${{ matrix.os == 'Windows' && '-DCMAKE_SUPPRESS_REGENERATION=ON' || '' }} - name: Build UMF run: cmake --build ${{env.BUILD_DIR}} --config ${{matrix.build_type}} -j ${{matrix.number_of_processors}} diff --git a/.github/workflows/reusable_multi_numa.yml b/.github/workflows/reusable_multi_numa.yml index 3c60bebc38..47a48adb20 100644 --- a/.github/workflows/reusable_multi_numa.yml +++ b/.github/workflows/reusable_multi_numa.yml @@ -1,4 +1,4 @@ -# Runs tests on multi-numa machine +# Runs tests on multi-numa machines name: MultiNuma on: [workflow_call] @@ -20,7 +20,7 @@ jobs: strategy: matrix: - os: [ubuntu-22.04, rhel-9.1] + os: [ubuntu-22.04, rhel-9.1, sles-15] build_type: [Debug, Release] shared_library: ['ON', 'OFF'] runs-on: ["DSS-MULTI-NUMA", "DSS-${{matrix.os}}"] @@ -31,9 +31,6 @@ jobs: with: fetch-depth: 0 - - name: Get information about platform - run: .github/scripts/get_system_info.sh - - name: Configure build run: > cmake @@ -53,16 +50,16 @@ jobs: run: cmake --build ${{github.workspace}}/build -j $(nproc) - name: Run tests - if: matrix.os != 'rhel-9.1' + if: (matrix.os != 'rhel-9.1') && (matrix.os != 'sles-15') working-directory: ${{github.workspace}}/build run: ctest --output-on-failure --test-dir test - # On RHEL, hwloc version is just a little too low. + # On RHEL/SLES, hwloc version is just a little too low. # Skip some tests until we upgrade hwloc and update CMake to properly handle local hwloc installation. # TODO: fix issue #560 # TODO: add issue for -E test_init_teardown - it is not clear why it fails - - name: Run tests (on RHEL) - if: matrix.os == 'rhel-9.1' + - name: Run tests (on RHEL/SLES) + if: (matrix.os == 'rhel-9.1') || (matrix.os == 'sles-15') working-directory: ${{github.workspace}}/build run: | ctest --output-on-failure --test-dir test -E "test_provider_os_memory_multiple_numa_nodes|test_init_teardown" @@ -70,7 +67,7 @@ jobs: --gtest_filter="-*checkModeLocal/*:*checkModePreferredEmptyNodeset/*:testNuma.checkModeInterleave" - name: Run NUMA tests under valgrind - if: matrix.os != 'rhel-9.1' + if: (matrix.os != 'rhel-9.1') && (matrix.os != 'sles-15') run: | ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{env.BUILD_DIR}} memcheck "${{env.NUMA_TESTS}}" ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{env.BUILD_DIR}} drd "${{env.NUMA_TESTS}}" @@ -91,3 +88,7 @@ jobs: with: name: ${{env.COVERAGE_NAME}}-${{matrix.os}}-shared-${{matrix.shared_library}} path: ${{env.COVERAGE_DIR}} + + - name: Get information about platform + if: always() + run: .github/scripts/get_system_info.sh diff --git a/.github/workflows/reusable_proxy_lib.yml b/.github/workflows/reusable_proxy_lib.yml index 363e665264..c519be95b8 100644 --- a/.github/workflows/reusable_proxy_lib.yml +++ b/.github/workflows/reusable_proxy_lib.yml @@ -32,7 +32,7 @@ jobs: - name: Install apt packages run: | sudo apt-get update - sudo apt-get install -y cmake libhwloc-dev libtbb-dev lcov + sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev lcov - name: Configure build run: > diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index 7b04f2061e..f57c0d5ae4 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -31,6 +31,8 @@ // Refer to the 'argsName()' function in each component to find detailed descriptions of these arguments. static void multithreaded(benchmark::internal::Benchmark *benchmark) { + benchmark->Threads(12); + benchmark->Threads(8); benchmark->Threads(4); benchmark->Threads(1); } @@ -165,15 +167,14 @@ UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, disjoint_pool_uniform_fixedprovider) ->Apply(&default_multiple_alloc_uniform_size) - ->Apply(&singlethreaded); -// TODO: change to multithreaded -//->Apply(&multithreaded); + ->Apply(&multithreaded); #ifdef UMF_POOL_JEMALLOC_ENABLED UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, jemalloc_pool_fixedprovider, fixed_alloc_size, pool_allocator>); -UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, jemalloc_pool_fix) +UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, + jemalloc_pool_fixedprovider) ->Apply(&default_multiple_alloc_fix_size) ->Apply(&multithreaded); @@ -181,7 +182,8 @@ UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, jemalloc_pool_uniform_fixedprovider, uniform_alloc_size, pool_allocator>); -UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, jemalloc_pool_uniform) +UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, + jemalloc_pool_uniform_fixedprovider) ->Apply(&default_multiple_alloc_uniform_size) ->Apply(&multithreaded); diff --git a/docs/config/api.rst b/docs/config/api.rst index 1c20d709c2..97e664d97f 100644 --- a/docs/config/api.rst +++ b/docs/config/api.rst @@ -168,6 +168,26 @@ IPC API allows retrieving IPC handles for the memory buffers allocated from UMF memory pools. The memory provider used by the pool should support IPC operations for this API to work. Otherwise IPC APIs return an error. +IPC caching +------------------------------------------ + +UMF employs IPC caching to avoid multiple IPC handles being created for the same +coarse-grain memory region allocated by the memory provider. UMF guarantees that +for each coarse-grain memory region allocated by the memory provider, only one +IPC handle is created when the :any:`umfGetIPCHandle` function is called. All +subsequent calls to the :any:`umfGetIPCHandle` function for the pointer to the +same memory region will return the entry from the cache. + +The same is true for the :any:`umfOpenIPCHandle` function. The actual mapping +of the IPC handle to the virtual address space is created only once, and all +subsequent calls to open the same IPC handle will return the entry from the cache. +The size of the cache for opened IPC handles is controlled by the ``UMF_MAX_OPENED_IPC_HANDLES`` +environment variable. By default, the cache size is unlimited. However, if the environment +variable is set and the cache size exceeds the limit, old items will be evicted. UMF tracks +the ref count for each entry in the cache and can evict only items with the ref count equal to 0. +The ref count is increased when the :any:`umfOpenIPCHandle` function is called and decreased +when the :any:`umfCloseIPCHandle` function is called for the corresponding IPC handle. + .. _ipc-api: IPC API diff --git a/include/umf/base.h b/include/umf/base.h index cc6b0ccbd7..0a12a319e6 100644 --- a/include/umf/base.h +++ b/include/umf/base.h @@ -47,7 +47,8 @@ typedef enum umf_result_t { 6, ///< Failure in user provider code (i.e in user provided callback) UMF_RESULT_ERROR_DEPENDENCY_UNAVAILABLE = 7, ///< External required dependency is unavailable or missing - UMF_RESULT_ERROR_UNKNOWN = 0x7ffffffe ///< Unknown or internal error + UMF_RESULT_ERROR_OUT_OF_RESOURCES = 8, ///< Out of internal resources + UMF_RESULT_ERROR_UNKNOWN = 0x7ffffffe ///< Unknown error } umf_result_t; /// @brief Type of the CTL query diff --git a/include/umf/pools/pool_disjoint.h b/include/umf/pools/pool_disjoint.h index d268a1dac4..a1558b85bf 100644 --- a/include/umf/pools/pool_disjoint.h +++ b/include/umf/pools/pool_disjoint.h @@ -100,7 +100,7 @@ umf_result_t umfDisjointPoolParamsSetSharedLimits( /// @brief Set custom name of the disjoint pool to be used in the traces. /// @param hParams handle to the parameters of the disjoint pool. -/// @param name custom name of the pool. +/// @param name custom name of the pool. Name longer than 64 characters will be truncated. /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. umf_result_t umfDisjointPoolParamsSetName(umf_disjoint_pool_params_handle_t hParams, diff --git a/include/umf/providers/provider_cuda.h b/include/umf/providers/provider_cuda.h index e3b81858bb..95f2634fbc 100644 --- a/include/umf/providers/provider_cuda.h +++ b/include/umf/providers/provider_cuda.h @@ -20,7 +20,8 @@ typedef struct umf_cuda_memory_provider_params_t *umf_cuda_memory_provider_params_handle_t; /// @brief Create a struct to store parameters of the CUDA Memory Provider. -/// @param hParams [out] handle to the newly created parameters struct. +/// @param hParams [out] handle to the newly created parameters structure, +/// initialized with the default (current) context and device ID. /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. umf_result_t umfCUDAMemoryProviderParamsCreate( umf_cuda_memory_provider_params_handle_t *hParams); diff --git a/scripts/qemu/configs/default.xml b/scripts/qemu/configs/default.xml index 5654687949..5d3198f60a 100644 --- a/scripts/qemu/configs/default.xml +++ b/scripts/qemu/configs/default.xml @@ -50,16 +50,12 @@ Cell 2 | 0 | 1200MiB | 17, 28, 10 | - - - - diff --git a/src/base_alloc/base_alloc.c b/src/base_alloc/base_alloc.c index 6f975307dc..00e58078e6 100644 --- a/src/base_alloc/base_alloc.c +++ b/src/base_alloc/base_alloc.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -230,6 +230,7 @@ void *umf_ba_alloc(umf_ba_pool_t *pool) { // check if the free list is not empty if (pool->metadata.free_list == NULL) { LOG_ERR("base_alloc: Free list should not be empty before new alloc"); + utils_mutex_unlock(&pool->metadata.free_lock); return NULL; } diff --git a/src/base_alloc/base_alloc_global.c b/src/base_alloc/base_alloc_global.c index f3b61566aa..ecec3367c9 100644 --- a/src/base_alloc/base_alloc_global.c +++ b/src/base_alloc/base_alloc_global.c @@ -71,7 +71,7 @@ static void umf_ba_create_global(void) { } size_t smallestSize = BASE_ALLOC.ac_sizes[0]; - BASE_ALLOC.smallest_ac_size_log2 = log2Utils(smallestSize); + BASE_ALLOC.smallest_ac_size_log2 = utils_msb64(smallestSize); LOG_DEBUG("UMF base allocator created"); } @@ -83,8 +83,8 @@ static int size_to_idx(size_t size) { } int isPowerOf2 = (0 == (size & (size - 1))); - int index = - (int)(log2Utils(size) + !isPowerOf2 - BASE_ALLOC.smallest_ac_size_log2); + int index = (int)(utils_msb64(size) + !isPowerOf2 - + BASE_ALLOC.smallest_ac_size_log2); assert(index >= 0); return index; diff --git a/src/base_alloc/base_alloc_linux.c b/src/base_alloc/base_alloc_linux.c index 260eec5aac..9b1dc63fe0 100644 --- a/src/base_alloc/base_alloc_linux.c +++ b/src/base_alloc/base_alloc_linux.c @@ -1,26 +1,22 @@ /* - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ #include -#include #include -#include #include -#include "base_alloc.h" -#include "base_alloc_global.h" #include "utils_concurrency.h" static UTIL_ONCE_FLAG Page_size_is_initialized = UTIL_ONCE_FLAG_INIT; static size_t Page_size; void *ba_os_alloc(size_t size) { - void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + void *ptr = utils_mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); // this should be unnecessary but pairs of mmap/munmap do not reset // asan's user-poisoning flags, leading to invalid error reports // Bug 81619: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81619 @@ -29,7 +25,7 @@ void *ba_os_alloc(size_t size) { } void ba_os_free(void *ptr, size_t size) { - int ret = munmap(ptr, size); + int ret = utils_munmap(ptr, size); assert(ret == 0); (void)ret; // unused } diff --git a/src/critnib/critnib.c b/src/critnib/critnib.c index c95637f20c..5625781d3f 100644 --- a/src/critnib/critnib.c +++ b/src/critnib/critnib.c @@ -64,6 +64,7 @@ #include "utils_assert.h" #include "utils_common.h" #include "utils_concurrency.h" +#include "utils_math.h" /* * A node that has been deleted is left untouched for this many delete @@ -367,7 +368,7 @@ int critnib_insert(struct critnib *c, word key, void *value, int update) { } /* and convert that to an index. */ - sh_t sh = utils_mssb_index(at) & (sh_t) ~(SLICE - 1); + sh_t sh = utils_msb64(at) & (sh_t) ~(SLICE - 1); struct critnib_node *m = alloc_node(c); if (!m) { @@ -524,7 +525,9 @@ find_predecessor(struct critnib_node *__restrict n) { while (1) { int nib; for (nib = NIB; nib >= 0; nib--) { - if (n->child[nib]) { + struct critnib_node *m; + utils_atomic_load_acquire_ptr((void **)&n->child[nib], (void **)&m); + if (m) { break; } } @@ -533,7 +536,12 @@ find_predecessor(struct critnib_node *__restrict n) { return NULL; } - n = n->child[nib]; + utils_atomic_load_acquire_ptr((void **)&n->child[nib], (void **)&n); + + if (!n) { + return NULL; + } + if (is_leaf(n)) { return to_leaf(n); } @@ -637,7 +645,9 @@ static struct critnib_leaf *find_successor(struct critnib_node *__restrict n) { while (1) { unsigned nib; for (nib = 0; nib <= NIB; nib++) { - if (n->child[nib]) { + struct critnib_node *m; + utils_atomic_load_acquire_ptr((void **)&n->child[nib], (void **)&m); + if (m) { break; } } @@ -646,7 +656,12 @@ static struct critnib_leaf *find_successor(struct critnib_node *__restrict n) { return NULL; } - n = n->child[nib]; + utils_atomic_load_acquire_ptr((void **)&n->child[nib], (void **)&n); + + if (!n) { + return NULL; + } + if (is_leaf(n)) { return to_leaf(n); } diff --git a/src/ipc.c b/src/ipc.c index 12c7bb9786..d4e5cc8066 100644 --- a/src/ipc.c +++ b/src/ipc.c @@ -146,19 +146,15 @@ umf_result_t umfOpenIPCHandle(umf_ipc_handler_handle_t hIPCHandler, } umf_result_t umfCloseIPCHandle(void *ptr) { - umf_alloc_info_t allocInfo; - umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); + umf_ipc_info_t ipcInfo; + umf_result_t ret = umfMemoryTrackerGetIpcInfo(ptr, &ipcInfo); if (ret != UMF_RESULT_SUCCESS) { - LOG_ERR("cannot get alloc info for ptr = %p.", ptr); + LOG_ERR("cannot get IPC info for ptr = %p.", ptr); return ret; } - // We cannot use umfPoolGetMemoryProvider function because it returns - // upstream provider but we need tracking one - umf_memory_provider_handle_t hProvider = allocInfo.pool->provider; - - return umfMemoryProviderCloseIPCHandle(hProvider, allocInfo.base, - allocInfo.baseSize); + return umfMemoryProviderCloseIPCHandle(ipcInfo.provider, ipcInfo.base, + ipcInfo.baseSize); } umf_result_t umfPoolGetIPCHandler(umf_memory_pool_handle_t hPool, diff --git a/src/ipc_cache.c b/src/ipc_cache.c index 6d5d39e4f2..bf17a66a42 100644 --- a/src/ipc_cache.c +++ b/src/ipc_cache.c @@ -54,6 +54,22 @@ typedef struct ipc_opened_cache_t { ipc_opened_cache_global_t *IPC_OPENED_CACHE_GLOBAL = NULL; +// Returns value of the UMF_MAX_OPENED_IPC_HANDLES environment variable +// or 0 if it is not set. +static size_t umfIpcCacheGlobalInitMaxOpenedHandles(void) { + const char *max_size_str = getenv("UMF_MAX_OPENED_IPC_HANDLES"); + if (max_size_str) { + char *endptr; + size_t max_size = strtoul(max_size_str, &endptr, 10); + if (*endptr == '\0') { + return max_size; + } + LOG_ERR("Invalid value of UMF_MAX_OPENED_IPC_HANDLES: %s", + max_size_str); + } + return 0; +} + umf_result_t umfIpcCacheGlobalInit(void) { umf_result_t ret = UMF_RESULT_SUCCESS; ipc_opened_cache_global_t *cache_global = @@ -78,8 +94,7 @@ umf_result_t umfIpcCacheGlobalInit(void) { goto err_mutex_destroy; } - // TODO: make max_size configurable via environment variable - cache_global->max_size = 0; + cache_global->max_size = umfIpcCacheGlobalInitMaxOpenedHandles(); cache_global->cur_size = 0; cache_global->lru_list = NULL; @@ -191,7 +206,19 @@ umf_result_t umfIpcOpenedCacheGet(ipc_opened_cache_handle_t cache, if (entry == NULL && cache->global->max_size != 0 && cache->global->cur_size >= cache->global->max_size) { // If max_size is set and the cache is full, evict the least recently used entry. - entry = cache->global->lru_list->prev; + // we need to search for the least recently used entry with ref_count == 0 + // The utlist implementation of the doubly-linked list keeps a tail pointer in head->prev + ipc_opened_cache_entry_t *candidate = cache->global->lru_list->prev; + do { + uint64_t ref_count = 0; + utils_atomic_load_acquire_u64(&candidate->ref_count, + &ref_count); + if (ref_count == 0) { + entry = candidate; + break; + } + candidate = candidate->prev; + } while (candidate != cache->global->lru_list->prev); } if (entry) { // we have eviction candidate @@ -244,3 +271,20 @@ umf_result_t umfIpcOpenedCacheGet(ipc_opened_cache_handle_t cache, return ret; } + +umf_result_t +umfIpcHandleMappedCacheRelease(ipc_opened_cache_value_t *cacheValue) { + if (!cacheValue) { + LOG_ERR("cacheValue is NULL"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + // get pointer to the entry + ipc_opened_cache_entry_t *entry = + (ipc_opened_cache_entry_t *)((char *)cacheValue - + offsetof(ipc_opened_cache_entry_t, value)); + // decrement the ref count + utils_atomic_decrement_u64(&entry->ref_count); + + return UMF_RESULT_SUCCESS; +} diff --git a/src/ipc_cache.h b/src/ipc_cache.h index 80870d3737..545c6e1e7e 100644 --- a/src/ipc_cache.h +++ b/src/ipc_cache.h @@ -47,4 +47,6 @@ umf_result_t umfIpcOpenedCacheGet(ipc_opened_cache_handle_t cache, uint64_t handle_id, ipc_opened_cache_value_t **retEntry); +umf_result_t +umfIpcHandleMappedCacheRelease(ipc_opened_cache_value_t *cacheValue); #endif /* UMF_IPC_CACHE_H */ diff --git a/src/memory_pool.c b/src/memory_pool.c index 1b61555de7..c948823340 100644 --- a/src/memory_pool.c +++ b/src/memory_pool.c @@ -141,6 +141,8 @@ void umfPoolDestroy(umf_memory_pool_handle_t hPool) { umf_result_t umfFree(void *ptr) { umf_memory_pool_handle_t hPool = umfPoolByPtr(ptr); if (hPool) { + LOG_DEBUG("calling umfPoolFree(pool=%p, ptr=%p) ...", (void *)hPool, + ptr); return umfPoolFree(hPool, ptr); } return UMF_RESULT_SUCCESS; diff --git a/src/pool/pool_disjoint.c b/src/pool/pool_disjoint.c index 0bd88bd246..ca4d2fab78 100644 --- a/src/pool/pool_disjoint.c +++ b/src/pool/pool_disjoint.c @@ -59,7 +59,7 @@ static __TLS umf_result_t TLS_last_allocation_error; // The largest size which is allocated via the allocator. // Allocations with size > CutOff bypass the pool and // go directly to the provider. -static size_t CutOff = (size_t)1 << 31; // 2GB +static const size_t CutOff = (size_t)1 << 31; // 2GB static size_t bucket_slab_min_size(bucket_t *bucket) { return bucket->pool->params.slab_min_size; @@ -75,28 +75,36 @@ static slab_t *create_slab(bucket_t *bucket) { umf_result_t res = UMF_RESULT_SUCCESS; umf_memory_provider_handle_t provider = bucket->pool->provider; - slab_t *slab = umf_ba_global_alloc(sizeof(*slab)); + size_t num_chunks_total = + utils_max(bucket_slab_min_size(bucket) / bucket->size, 1); + + // Calculate the number of 64-bit words needed. + size_t num_words = + (num_chunks_total + CHUNK_BITMAP_SIZE - 1) / CHUNK_BITMAP_SIZE; + + slab_t *slab = umf_ba_global_alloc(sizeof(*slab) + + num_words * sizeof(slab->chunks[0])); if (slab == NULL) { LOG_ERR("allocation of new slab failed!"); return NULL; } slab->num_chunks_allocated = 0; - slab->first_free_chunk_idx = 0; slab->bucket = bucket; slab->iter.val = slab; slab->iter.prev = slab->iter.next = NULL; - slab->num_chunks_total = - utils_max(bucket_slab_min_size(bucket) / bucket->size, 1); - slab->chunks = - umf_ba_global_alloc(sizeof(*slab->chunks) * slab->num_chunks_total); - if (slab->chunks == NULL) { - LOG_ERR("allocation of slab chunks failed!"); - goto free_slab; + slab->num_chunks_total = num_chunks_total; + slab->num_words = num_words; + + // set all chunks as free + memset(slab->chunks, ~0, num_words * sizeof(slab->chunks[0])); + if (num_chunks_total % CHUNK_BITMAP_SIZE) { + // clear remaining bits + slab->chunks[num_words - 1] = + ((1ULL << (num_chunks_total % CHUNK_BITMAP_SIZE)) - 1); } - memset(slab->chunks, 0, sizeof(*slab->chunks) * slab->num_chunks_total); // if slab_min_size is not a multiple of bucket size, we would have some // padding at the end of the slab @@ -108,7 +116,7 @@ static slab_t *create_slab(bucket_t *bucket) { res = umfMemoryProviderAlloc(provider, slab->slab_size, 0, &slab->mem_ptr); if (res != UMF_RESULT_SUCCESS) { LOG_ERR("allocation of slab data failed!"); - goto free_slab_chunks; + goto free_slab; } // raw allocation is not available for user so mark it as inaccessible @@ -117,9 +125,6 @@ static slab_t *create_slab(bucket_t *bucket) { LOG_DEBUG("bucket: %p, slab_size: %zu", (void *)bucket, slab->slab_size); return slab; -free_slab_chunks: - umf_ba_global_free(slab->chunks); - free_slab: umf_ba_global_free(slab); return NULL; @@ -136,25 +141,21 @@ static void destroy_slab(slab_t *slab) { LOG_ERR("deallocation of slab data failed!"); } - umf_ba_global_free(slab->chunks); umf_ba_global_free(slab); } -// return the index of the first available chunk, SIZE_MAX otherwise static size_t slab_find_first_available_chunk_idx(const slab_t *slab) { - // use the first free chunk index as a hint for the search - for (bool *chunk = slab->chunks + slab->first_free_chunk_idx; - chunk != slab->chunks + slab->num_chunks_total; chunk++) { - - // false means not used - if (*chunk == false) { - size_t idx = chunk - slab->chunks; - LOG_DEBUG("idx: %zu", idx); - return idx; + for (size_t i = 0; i < slab->num_words; i++) { + // NOTE: free chunks are represented as set bits + uint64_t word = slab->chunks[i]; + if (word != 0) { + size_t bit_index = utils_lsb64(word); + size_t free_chunk = i * CHUNK_BITMAP_SIZE + bit_index; + return free_chunk; } } - LOG_DEBUG("idx: SIZE_MAX"); + // No free chunk was found. return SIZE_MAX; } @@ -167,12 +168,9 @@ static void *slab_get_chunk(slab_t *slab) { (void *)((uintptr_t)slab->mem_ptr + chunk_idx * slab->bucket->size); // mark chunk as used - slab->chunks[chunk_idx] = true; + slab_set_chunk_bit(slab, chunk_idx, false); slab->num_chunks_allocated += 1; - // use the found index as the next hint - slab->first_free_chunk_idx = chunk_idx + 1; - return free_chunk; } @@ -195,18 +193,9 @@ static void slab_free_chunk(slab_t *slab, void *ptr) { size_t chunk_idx = ptr_diff / slab->bucket->size; // Make sure that the chunk was allocated - assert(slab->chunks[chunk_idx] && "double free detected"); - slab->chunks[chunk_idx] = false; + assert(slab_read_chunk_bit(slab, chunk_idx) == 0 && "double free detected"); + slab_set_chunk_bit(slab, chunk_idx, true); slab->num_chunks_allocated -= 1; - - if (chunk_idx < slab->first_free_chunk_idx) { - slab->first_free_chunk_idx = chunk_idx; - } - - LOG_DEBUG("chunk_idx: %zu, num_chunks_allocated: %zu, " - "first_free_chunk_idx: %zu", - chunk_idx, slab->num_chunks_allocated, - slab->first_free_chunk_idx); } static bool slab_has_avail(const slab_t *slab) { @@ -466,9 +455,9 @@ static size_t size_to_idx(disjoint_pool_t *pool, size_t size) { } // get the position of the leftmost set bit - size_t position = getLeftmostSetBitPos(size); + size_t position = utils_msb64(size); - bool is_power_of_2 = 0 == (size & (size - 1)); + bool is_power_of_2 = IS_POWER_OF_2(size); bool larger_than_halfway_between_powers_of_2 = !is_power_of_2 && (bool)((size - 1) & ((uint64_t)(1) << (position - 1))); @@ -588,12 +577,6 @@ umf_result_t disjoint_pool_initialize(umf_memory_provider_handle_t provider, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - disjoint_pool_t *disjoint_pool = - umf_ba_global_alloc(sizeof(*disjoint_pool)); - if (!disjoint_pool) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - umf_disjoint_pool_params_t *dp_params = (umf_disjoint_pool_params_t *)params; @@ -604,12 +587,21 @@ umf_result_t disjoint_pool_initialize(umf_memory_provider_handle_t provider, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } + disjoint_pool_t *disjoint_pool = + umf_ba_global_alloc(sizeof(*disjoint_pool)); + if (disjoint_pool == NULL) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + VALGRIND_DO_CREATE_MEMPOOL(disjoint_pool, 0, 0); disjoint_pool->provider = provider; disjoint_pool->params = *dp_params; disjoint_pool->known_slabs = critnib_new(); + if (disjoint_pool->known_slabs == NULL) { + goto err_free_disjoint_pool; + } // Generate buckets sized such as: 64, 96, 128, 192, ..., CutOff. // Powers of 2 and the value halfway between the powers of 2. @@ -622,21 +614,29 @@ umf_result_t disjoint_pool_initialize(umf_memory_provider_handle_t provider, Size1 = utils_max(Size1, UMF_DISJOINT_POOL_MIN_BUCKET_DEFAULT_SIZE); // Calculate the exponent for min_bucket_size used for finding buckets. - disjoint_pool->min_bucket_size_exp = (size_t)log2Utils(Size1); + disjoint_pool->min_bucket_size_exp = (size_t)utils_msb64(Size1); disjoint_pool->default_shared_limits = umfDisjointPoolSharedLimitsCreate(SIZE_MAX); + if (disjoint_pool->default_shared_limits == NULL) { + goto err_free_known_slabs; + } // count number of buckets, start from 1 disjoint_pool->buckets_num = 1; size_t Size2 = Size1 + Size1 / 2; size_t ts2 = Size2, ts1 = Size1; - for (; Size2 < CutOff; Size1 *= 2, Size2 *= 2) { + while (Size2 < CutOff) { disjoint_pool->buckets_num += 2; + Size2 *= 2; } + disjoint_pool->buckets = umf_ba_global_alloc( sizeof(*disjoint_pool->buckets) * disjoint_pool->buckets_num); + if (disjoint_pool->buckets == NULL) { + goto err_free_shared_limits; + } - int i = 0; + size_t i = 0; Size1 = ts1; Size2 = ts2; for (; Size2 < CutOff; Size1 *= 2, Size2 *= 2, i += 2) { @@ -648,6 +648,13 @@ umf_result_t disjoint_pool_initialize(umf_memory_provider_handle_t provider, disjoint_pool->buckets[i] = create_bucket( CutOff, disjoint_pool, disjoint_pool_get_limits(disjoint_pool)); + // check if all buckets were created successfully + for (i = 0; i < disjoint_pool->buckets_num; i++) { + if (disjoint_pool->buckets[i] == NULL) { + goto err_free_buckets; + } + } + umf_result_t ret = umfMemoryProviderGetMinPageSize( provider, NULL, &disjoint_pool->provider_min_page_size); if (ret != UMF_RESULT_SUCCESS) { @@ -657,6 +664,25 @@ umf_result_t disjoint_pool_initialize(umf_memory_provider_handle_t provider, *ppPool = (void *)disjoint_pool; return UMF_RESULT_SUCCESS; + +err_free_buckets: + for (i = 0; i < disjoint_pool->buckets_num; i++) { + if (disjoint_pool->buckets[i] != NULL) { + destroy_bucket(disjoint_pool->buckets[i]); + } + } + umf_ba_global_free(disjoint_pool->buckets); + +err_free_shared_limits: + umfDisjointPoolSharedLimitsDestroy(disjoint_pool->default_shared_limits); + +err_free_known_slabs: + critnib_delete(disjoint_pool->known_slabs); + +err_free_disjoint_pool: + umf_ba_global_free(disjoint_pool); + + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } void *disjoint_pool_malloc(void *pool, size_t size) { @@ -767,6 +793,14 @@ void *disjoint_pool_aligned_malloc(void *pool, size_t size, size_t alignment) { return aligned_ptr; } +static size_t get_chunk_idx(void *ptr, slab_t *slab) { + return (((uintptr_t)ptr - (uintptr_t)slab->mem_ptr) / slab->bucket->size); +} + +static void *get_unaligned_ptr(size_t chunk_idx, slab_t *slab) { + return (void *)((uintptr_t)slab->mem_ptr + chunk_idx * slab->bucket->size); +} + size_t disjoint_pool_malloc_usable_size(void *pool, void *ptr) { disjoint_pool_t *disjoint_pool = (disjoint_pool_t *)pool; if (ptr == NULL) { @@ -788,10 +822,8 @@ size_t disjoint_pool_malloc_usable_size(void *pool, void *ptr) { } // Get the unaligned pointer // NOTE: the base pointer slab->mem_ptr needn't to be aligned to bucket size - size_t chunk_idx = - (((uintptr_t)ptr - (uintptr_t)slab->mem_ptr) / slab->bucket->size); - void *unaligned_ptr = - (void *)((uintptr_t)slab->mem_ptr + chunk_idx * slab->bucket->size); + size_t chunk_idx = get_chunk_idx(ptr, slab); + void *unaligned_ptr = get_unaligned_ptr(chunk_idx, slab); ptrdiff_t diff = (ptrdiff_t)ptr - (ptrdiff_t)unaligned_ptr; @@ -847,10 +879,8 @@ umf_result_t disjoint_pool_free(void *pool, void *ptr) { // Get the unaligned pointer // NOTE: the base pointer slab->mem_ptr needn't to be aligned to bucket size - size_t chunk_idx = - (((uintptr_t)ptr - (uintptr_t)slab->mem_ptr) / slab->bucket->size); - void *unaligned_ptr = - (void *)((uintptr_t)slab->mem_ptr + chunk_idx * slab->bucket->size); + size_t chunk_idx = get_chunk_idx(ptr, slab); + void *unaligned_ptr = get_unaligned_ptr(chunk_idx, slab); utils_annotate_memory_inaccessible(unaligned_ptr, bucket->size); bucket_free_chunk(bucket, unaligned_ptr, slab, &to_pool); @@ -876,13 +906,11 @@ umf_result_t disjoint_pool_free(void *pool, void *ptr) { umf_result_t disjoint_pool_get_last_allocation_error(void *pool) { (void)pool; - return TLS_last_allocation_error; } // Define destructor for use with unique_ptr void disjoint_pool_finalize(void *pool) { - disjoint_pool_t *hPool = (disjoint_pool_t *)pool; if (hPool->params.pool_trace > 1) { @@ -937,7 +965,7 @@ void umfDisjointPoolSharedLimitsDestroy( umf_result_t umfDisjointPoolParamsCreate(umf_disjoint_pool_params_handle_t *hParams) { - static const char *DEFAULT_NAME = "disjoint_pool"; + static char *DEFAULT_NAME = "disjoint_pool"; if (!hParams) { LOG_ERR("disjoint pool params handle is NULL"); @@ -951,20 +979,16 @@ umfDisjointPoolParamsCreate(umf_disjoint_pool_params_handle_t *hParams) { return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - params->slab_min_size = 0; - params->max_poolable_size = 0; - params->capacity = 0; - params->min_bucket_size = UMF_DISJOINT_POOL_MIN_BUCKET_DEFAULT_SIZE; - params->cur_pool_size = 0; - params->pool_trace = 0; - params->shared_limits = NULL; - params->name = NULL; - - umf_result_t ret = umfDisjointPoolParamsSetName(params, DEFAULT_NAME); - if (ret != UMF_RESULT_SUCCESS) { - umf_ba_global_free(params); - return ret; - } + *params = (umf_disjoint_pool_params_t){ + .slab_min_size = 0, + .max_poolable_size = 0, + .capacity = 0, + .min_bucket_size = UMF_DISJOINT_POOL_MIN_BUCKET_DEFAULT_SIZE, + .cur_pool_size = 0, + .pool_trace = 0, + .shared_limits = NULL, + .name = {*DEFAULT_NAME}, + }; *hParams = params; @@ -975,7 +999,6 @@ umf_result_t umfDisjointPoolParamsDestroy(umf_disjoint_pool_params_handle_t hParams) { // NOTE: dereferencing hParams when BA is already destroyed leads to crash if (hParams && !umf_ba_global_is_destroyed()) { - umf_ba_global_free(hParams->name); umf_ba_global_free(hParams); } @@ -1067,15 +1090,6 @@ umfDisjointPoolParamsSetName(umf_disjoint_pool_params_handle_t hParams, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - char *newName = umf_ba_global_alloc(sizeof(*newName) * (strlen(name) + 1)); - if (newName == NULL) { - LOG_ERR("cannot allocate memory for disjoint pool name"); - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - umf_ba_global_free(hParams->name); - hParams->name = newName; - strcpy(hParams->name, name); - + strncpy(hParams->name, name, sizeof(hParams->name) - 1); return UMF_RESULT_SUCCESS; } diff --git a/src/pool/pool_disjoint_internal.h b/src/pool/pool_disjoint_internal.h index 2b5de64bc3..7a63dd72ac 100644 --- a/src/pool/pool_disjoint_internal.h +++ b/src/pool/pool_disjoint_internal.h @@ -15,6 +15,8 @@ #include "critnib/critnib.h" #include "utils_concurrency.h" +#define CHUNK_BITMAP_SIZE 64 + typedef struct bucket_t bucket_t; typedef struct slab_t slab_t; typedef struct slab_list_item_t slab_list_item_t; @@ -81,23 +83,24 @@ typedef struct slab_t { void *mem_ptr; size_t slab_size; - // Represents the current state of each chunk: if the bit is set, the - // chunk is allocated; otherwise, the chunk is free for allocation - bool *chunks; size_t num_chunks_total; + // Num of 64-bit words needed to store chunk state + size_t num_words; + // Total number of allocated chunks at the moment. size_t num_chunks_allocated; // The bucket which the slab belongs to bucket_t *bucket; - // Hints where to start search for free chunk in a slab - size_t first_free_chunk_idx; - // Store iterator to the corresponding node in avail/unavail list // to achieve O(1) removal slab_list_item_t iter; + + // Represents the current state of each chunk: if the bit is clear, the + // chunk is allocated; otherwise, the chunk is free for allocation + uint64_t chunks[]; } slab_t; typedef struct umf_disjoint_pool_shared_limits_t { @@ -131,7 +134,7 @@ typedef struct umf_disjoint_pool_params_t { umf_disjoint_pool_shared_limits_handle_t shared_limits; // Name used in traces - char *name; + char name[64]; } umf_disjoint_pool_params_t; typedef struct disjoint_pool_t { @@ -158,4 +161,24 @@ typedef struct disjoint_pool_t { size_t provider_min_page_size; } disjoint_pool_t; +static inline void slab_set_chunk_bit(slab_t *slab, size_t index, bool value) { + assert(index < slab->num_chunks_total && "Index out of range"); + + size_t word_index = index / CHUNK_BITMAP_SIZE; + unsigned bit_index = index % CHUNK_BITMAP_SIZE; + if (value) { + slab->chunks[word_index] |= (1ULL << bit_index); + } else { + slab->chunks[word_index] &= ~(1ULL << bit_index); + } +} + +static inline int slab_read_chunk_bit(const slab_t *slab, size_t index) { + assert(index < slab->num_chunks_total && "Index out of range"); + + size_t word_index = index / CHUNK_BITMAP_SIZE; + unsigned bit_index = index % CHUNK_BITMAP_SIZE; + return (slab->chunks[word_index] >> bit_index) & 1; +} + #endif // UMF_POOL_DISJOINT_INTERNAL_H diff --git a/src/provider/provider_cuda.c b/src/provider/provider_cuda.c index bb4b3cf644..dd12d91847 100644 --- a/src/provider/provider_cuda.c +++ b/src/provider/provider_cuda.c @@ -139,6 +139,7 @@ typedef struct cu_ops_t { CUresult (*cuGetErrorName)(CUresult error, const char **pStr); CUresult (*cuGetErrorString)(CUresult error, const char **pStr); CUresult (*cuCtxGetCurrent)(CUcontext *pctx); + CUresult (*cuCtxGetDevice)(CUdevice *device); CUresult (*cuCtxSetCurrent)(CUcontext ctx); CUresult (*cuIpcGetMemHandle)(CUipcMemHandle *pHandle, CUdeviceptr dptr); CUresult (*cuIpcOpenMemHandle)(CUdeviceptr *pdptr, CUipcMemHandle handle, @@ -178,6 +179,9 @@ static umf_result_t cu2umf_result(CUresult result) { case CUDA_ERROR_INVALID_VALUE: case CUDA_ERROR_INVALID_HANDLE: return UMF_RESULT_ERROR_INVALID_ARGUMENT; + case CUDA_ERROR_DEINITIALIZED: + LOG_ERR("CUDA driver has been deinitialized"); + return UMF_RESULT_ERROR_OUT_OF_RESOURCES; default: cu_store_last_native_error(result); return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; @@ -221,6 +225,8 @@ static void init_cu_global_state(void) { utils_get_symbol_addr(lib_handle, "cuGetErrorString", lib_name); *(void **)&g_cu_ops.cuCtxGetCurrent = utils_get_symbol_addr(lib_handle, "cuCtxGetCurrent", lib_name); + *(void **)&g_cu_ops.cuCtxGetDevice = + utils_get_symbol_addr(lib_handle, "cuCtxGetDevice", lib_name); *(void **)&g_cu_ops.cuCtxSetCurrent = utils_get_symbol_addr(lib_handle, "cuCtxSetCurrent", lib_name); *(void **)&g_cu_ops.cuIpcGetMemHandle = @@ -234,9 +240,9 @@ static void init_cu_global_state(void) { !g_cu_ops.cuMemHostAlloc || !g_cu_ops.cuMemAllocManaged || !g_cu_ops.cuMemFree || !g_cu_ops.cuMemFreeHost || !g_cu_ops.cuGetErrorName || !g_cu_ops.cuGetErrorString || - !g_cu_ops.cuCtxGetCurrent || !g_cu_ops.cuCtxSetCurrent || - !g_cu_ops.cuIpcGetMemHandle || !g_cu_ops.cuIpcOpenMemHandle || - !g_cu_ops.cuIpcCloseMemHandle) { + !g_cu_ops.cuCtxGetCurrent || !g_cu_ops.cuCtxGetDevice || + !g_cu_ops.cuCtxSetCurrent || !g_cu_ops.cuIpcGetMemHandle || + !g_cu_ops.cuIpcOpenMemHandle || !g_cu_ops.cuIpcCloseMemHandle) { LOG_FATAL("Required CUDA symbols not found."); Init_cu_global_state_failed = true; utils_close_library(lib_handle); @@ -260,8 +266,29 @@ umf_result_t umfCUDAMemoryProviderParamsCreate( return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - params_data->cuda_context_handle = NULL; - params_data->cuda_device_handle = -1; + utils_init_once(&cu_is_initialized, init_cu_global_state); + if (Init_cu_global_state_failed) { + LOG_FATAL("Loading CUDA symbols failed"); + return UMF_RESULT_ERROR_DEPENDENCY_UNAVAILABLE; + } + + // initialize context and device to the current ones + CUcontext current_ctx = NULL; + CUresult cu_result = g_cu_ops.cuCtxGetCurrent(¤t_ctx); + if (cu_result == CUDA_SUCCESS) { + params_data->cuda_context_handle = current_ctx; + } else { + params_data->cuda_context_handle = NULL; + } + + CUdevice current_device = -1; + cu_result = g_cu_ops.cuCtxGetDevice(¤t_device); + if (cu_result == CUDA_SUCCESS) { + params_data->cuda_device_handle = current_device; + } else { + params_data->cuda_device_handle = -1; + } + params_data->memory_type = UMF_MEMORY_TYPE_UNKNOWN; params_data->alloc_flags = 0; @@ -342,6 +369,12 @@ static umf_result_t cu_memory_provider_initialize(void *params, } if (cu_params->cuda_context_handle == NULL) { + LOG_ERR("Invalid context handle"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (cu_params->cuda_device_handle < 0) { + LOG_ERR("Invalid device handle"); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } @@ -539,22 +572,41 @@ static void cu_memory_provider_get_last_native_error(void *provider, return; } - const char *error_name = 0; - const char *error_string = 0; - g_cu_ops.cuGetErrorName(TLS_last_native_error.native_error, &error_name); - g_cu_ops.cuGetErrorString(TLS_last_native_error.native_error, - &error_string); - + CUresult result; size_t buf_size = 0; - strncpy(TLS_last_native_error.msg_buff, error_name, TLS_MSG_BUF_LEN - 1); - buf_size = strlen(TLS_last_native_error.msg_buff); + const char *error_name = NULL; + const char *error_string = NULL; + + // If the error code is not recognized, + // CUDA_ERROR_INVALID_VALUE will be returned + // and error_name will be set to the NULL address. + result = g_cu_ops.cuGetErrorName(TLS_last_native_error.native_error, + &error_name); + if (result == CUDA_SUCCESS && error_name != NULL) { + strncpy(TLS_last_native_error.msg_buff, error_name, + TLS_MSG_BUF_LEN - 1); + } else { + strncpy(TLS_last_native_error.msg_buff, "cuGetErrorName() failed", + TLS_MSG_BUF_LEN - 1); + } + buf_size = strlen(TLS_last_native_error.msg_buff); strncat(TLS_last_native_error.msg_buff, " - ", TLS_MSG_BUF_LEN - buf_size - 1); buf_size = strlen(TLS_last_native_error.msg_buff); - strncat(TLS_last_native_error.msg_buff, error_string, - TLS_MSG_BUF_LEN - buf_size - 1); + // If the error code is not recognized, + // CUDA_ERROR_INVALID_VALUE will be returned + // and error_string will be set to the NULL address. + result = g_cu_ops.cuGetErrorString(TLS_last_native_error.native_error, + &error_string); + if (result == CUDA_SUCCESS && error_string != NULL) { + strncat(TLS_last_native_error.msg_buff, error_string, + TLS_MSG_BUF_LEN - buf_size - 1); + } else { + strncat(TLS_last_native_error.msg_buff, "cuGetErrorString() failed", + TLS_MSG_BUF_LEN - buf_size - 1); + } *pError = TLS_last_native_error.native_error; *ppMessage = TLS_last_native_error.msg_buff; diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index 4696bc562d..c5a4b5f1f4 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -7,73 +7,229 @@ * */ -#include "provider_tracking.h" +#include +#include +#include +#include +#include + +#include +#include +#include + #include "base_alloc_global.h" #include "critnib.h" #include "ipc_cache.h" #include "ipc_internal.h" +#include "memory_pool_internal.h" +#include "provider_tracking.h" #include "utils_common.h" #include "utils_concurrency.h" #include "utils_log.h" -#include -#include -#include - -#include -#include -#include -#include -#include +// TODO: we need to support an arbitrary amount of layers in the future +#define MAX_LEVELS_OF_ALLOC_SEGMENT_MAP 8 uint64_t IPC_HANDLE_ID = 0; struct umf_memory_tracker_t { umf_ba_pool_t *alloc_info_allocator; - critnib *alloc_segments_map; + // Multilevel maps are needed to support the case + // when one memory pool acts as a memory provider + // for another memory pool (nested memory pooling). + critnib *alloc_segments_map[MAX_LEVELS_OF_ALLOC_SEGMENT_MAP]; utils_mutex_t splitMergeMutex; + umf_ba_pool_t *ipc_info_allocator; + critnib *ipc_segments_map; }; typedef struct tracker_alloc_info_t { umf_memory_pool_handle_t pool; size_t size; + // number of overlapping memory regions + // in the next level of map + // falling within the current range + size_t n_children; } tracker_alloc_info_t; -static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, - umf_memory_pool_handle_t pool, - const void *ptr, size_t size) { +typedef struct tracker_ipc_info_t { + size_t size; + umf_memory_provider_handle_t provider; + ipc_opened_cache_value_t *ipc_cache_value; +} tracker_ipc_info_t; + +// Get the most nested (on the highest level) allocation segment in the map with the `ptr` key. +// If `no_children` is set to 1, the function will return the entry +// only if it has no children on the higher level. +// The function returns the entry if found, otherwise NULL. +static tracker_alloc_info_t *get_most_nested_alloc_segment( + umf_memory_tracker_handle_t hTracker, const void *ptr, int *_level, + uintptr_t *_parent_key, tracker_alloc_info_t **_parent_value, + int no_children) { + assert(ptr); + + tracker_alloc_info_t *parent_value = NULL; + tracker_alloc_info_t *rvalue = NULL; + uintptr_t parent_key = 0; + uintptr_t rkey = 0; + uint64_t rsize = 0; + int level = 0; + int found = 0; + + do { + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + found = + critnib_find(hTracker->alloc_segments_map[level], (uintptr_t)ptr, + FIND_LE, (void *)&rkey, (void **)&rvalue); + if (!found || !rvalue) { + break; + } + + utils_atomic_load_acquire_u64((uint64_t *)&rvalue->size, &rsize); + + if (found && (uintptr_t)ptr < rkey + rsize) { + if (rvalue->n_children) { + if (level == MAX_LEVELS_OF_ALLOC_SEGMENT_MAP - 1) { + break; + } + level++; + parent_key = rkey; + parent_value = rvalue; + } + } + } while (found && ((uintptr_t)ptr < rkey + rsize) && rvalue->n_children); + + if (!rvalue || rkey != (uintptr_t)ptr) { + return NULL; + } + + if (no_children && (rvalue->n_children > 0)) { + return NULL; + } + + if (_level) { + *_level = level; + } + if (_parent_key) { + *_parent_key = parent_key; + } + if (_parent_value) { + *_parent_value = parent_value; + } + + assert(!no_children || rvalue->n_children == 0); + + return rvalue; +} + +static umf_result_t +umfMemoryTrackerAddAtLevel(umf_memory_tracker_handle_t hTracker, int level, + umf_memory_pool_handle_t pool, const void *ptr, + size_t size, uintptr_t parent_key, + tracker_alloc_info_t *parent_value) { assert(ptr); + umf_result_t umf_result = UMF_RESULT_ERROR_UNKNOWN; + tracker_alloc_info_t *value = umf_ba_alloc(hTracker->alloc_info_allocator); if (value == NULL) { - LOG_ERR("failed to allocate tracker value, ptr=%p, size=%zu", ptr, + LOG_ERR("failed to allocate a tracker value, ptr=%p, size=%zu", ptr, size); return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } value->pool = pool; - value->size = size; - - int ret = - critnib_insert(hTracker->alloc_segments_map, (uintptr_t)ptr, value, 0); + utils_atomic_store_release_u64(&value->size, size); + value->n_children = 0; + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + int ret = critnib_insert(hTracker->alloc_segments_map[level], + (uintptr_t)ptr, value, 0); if (ret == 0) { - LOG_DEBUG( - "memory region is added, tracker=%p, ptr=%p, pool=%p, size=%zu", - (void *)hTracker, ptr, (void *)pool, size); + LOG_DEBUG("memory region is added, tracker=%p, level=%i, pool=%p, " + "ptr=%p, size=%zu", + (void *)hTracker, level, (void *)pool, ptr, size); + + if (parent_value) { + parent_value->n_children++; + LOG_DEBUG( + "child #%zu added to memory region: tracker=%p, level=%i, " + "pool=%p, ptr=%p, size=%zu", + parent_value->n_children, (void *)hTracker, level - 1, + (void *)parent_value->pool, (void *)parent_key, + parent_value->size); + } return UMF_RESULT_SUCCESS; } + if (ret == ENOMEM) { + umf_result = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } - LOG_ERR("failed to insert tracker value, ret=%d, ptr=%p, pool=%p, size=%zu", - ret, ptr, (void *)pool, size); + LOG_ERR( + "failed to insert the tracker value: pool=%p, ptr=%p, size=%zu, ret=%d", + (void *)pool, ptr, size, ret); umf_ba_free(hTracker->alloc_info_allocator, value); - if (ret == ENOMEM) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + return umf_result; +} + +static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, + umf_memory_pool_handle_t pool, + const void *ptr, size_t size) { + assert(ptr); + + umf_result_t umf_result = UMF_RESULT_ERROR_UNKNOWN; + tracker_alloc_info_t *parent_value = NULL; + tracker_alloc_info_t *rvalue = NULL; + uintptr_t parent_key = 0; + uintptr_t rkey = 0; + uint64_t rsize = 0; + int level = 0; + int found = 0; + + // Find the most nested (in the highest level) entry + // in the critnib maps that contains the given 'ptr' pointer. + do { + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + found = + critnib_find(hTracker->alloc_segments_map[level], (uintptr_t)ptr, + FIND_LE, (void *)&rkey, (void **)&rvalue); + if (!found || !rvalue) { + break; + } + + utils_atomic_load_acquire_u64((uint64_t *)&rvalue->size, &rsize); + + if ((uintptr_t)ptr < rkey + rsize) { + if (level == MAX_LEVELS_OF_ALLOC_SEGMENT_MAP - 1) { + // TODO: we need to support an arbitrary amount of layers in the future + LOG_ERR("tracker level is too high, ptr=%p, size=%zu", ptr, + size); + return UMF_RESULT_ERROR_OUT_OF_RESOURCES; + } + if (((uintptr_t)ptr + size) > (rkey + rsize)) { + LOG_ERR( + "cannot insert to the tracker value (pool=%p, ptr=%p, " + "size=%zu) " + "that exceeds the parent value (pool=%p, ptr=%p, size=%zu)", + (void *)pool, ptr, size, (void *)rvalue->pool, (void *)rkey, + (size_t)rsize); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + parent_key = rkey; + parent_value = rvalue; + level++; + } + } while (found && ((uintptr_t)ptr < rkey + rsize) && rvalue->n_children); + + umf_result = umfMemoryTrackerAddAtLevel(hTracker, level, pool, ptr, size, + parent_key, parent_value); + if (umf_result != UMF_RESULT_SUCCESS) { + return umf_result; } - return UMF_RESULT_ERROR_UNKNOWN; + return UMF_RESULT_SUCCESS; } static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, @@ -85,22 +241,107 @@ static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, // Every umfMemoryTrackerAdd(..., ptr, ...) should have a corresponding // umfMemoryTrackerRemove call with the same ptr value. - void *value = critnib_remove(hTracker->alloc_segments_map, (uintptr_t)ptr); + tracker_alloc_info_t *parent_value = NULL; + uintptr_t parent_key = 0; + int level = 0; + + // Find the most nested (on the highest level) entry in the map + // with the `ptr` key and with no children - only such entry can be removed. + tracker_alloc_info_t *value = get_most_nested_alloc_segment( + hTracker, ptr, &level, &parent_key, &parent_value, 1 /* no_children */); if (!value) { LOG_ERR("pointer %p not found in the alloc_segments_map", ptr); return UMF_RESULT_ERROR_UNKNOWN; } - tracker_alloc_info_t *v = value; + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + value = critnib_remove(hTracker->alloc_segments_map[level], (uintptr_t)ptr); + assert(value); + + LOG_DEBUG("memory region removed: tracker=%p, level=%i, pool=%p, ptr=%p, " + "size=%zu", + (void *)hTracker, level, value->pool, ptr, value->size); - LOG_DEBUG("memory region removed: tracker=%p, ptr=%p, size=%zu", - (void *)hTracker, ptr, v->size); + if (parent_value) { + LOG_DEBUG( + "child #%zu removed from memory region: tracker=%p, level=%i, " + "pool=%p, ptr=%p, size=%zu", + parent_value->n_children, (void *)hTracker, level - 1, + (void *)parent_value->pool, (void *)parent_key, parent_value->size); + parent_value->n_children--; + } umf_ba_free(hTracker->alloc_info_allocator, value); return UMF_RESULT_SUCCESS; } +static umf_result_t +umfMemoryTrackerAddIpcSegment(umf_memory_tracker_handle_t hTracker, + const void *ptr, size_t size, + umf_memory_provider_handle_t provider, + ipc_opened_cache_value_t *cache_entry) { + assert(hTracker); + assert(provider); + assert(cache_entry); + + tracker_ipc_info_t *value = umf_ba_alloc(hTracker->ipc_info_allocator); + + if (value == NULL) { + LOG_ERR("failed to allocate tracker_ipc_info_t, ptr=%p, size=%zu", ptr, + size); + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + value->size = size; + value->provider = provider; + value->ipc_cache_value = cache_entry; + + int ret = + critnib_insert(hTracker->ipc_segments_map, (uintptr_t)ptr, value, 0); + if (ret == 0) { + LOG_DEBUG("IPC memory region is added, tracker=%p, ptr=%p, size=%zu, " + "provider=%p, cache_entry=%p", + (void *)hTracker, ptr, size, provider, cache_entry); + return UMF_RESULT_SUCCESS; + } + + LOG_ERR("failed to insert tracker_ipc_info_t, ret=%d, ptr=%p, size=%zu, " + "provider=%p, cache_entry=%p", + ret, ptr, size, provider, cache_entry); + + umf_ba_free(hTracker->ipc_info_allocator, value); + + if (ret == ENOMEM) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + return UMF_RESULT_ERROR_UNKNOWN; +} + +static umf_result_t +umfMemoryTrackerRemoveIpcSegment(umf_memory_tracker_handle_t hTracker, + const void *ptr) { + assert(ptr); + + void *value = critnib_remove(hTracker->ipc_segments_map, (uintptr_t)ptr); + + if (!value) { + LOG_ERR("pointer %p not found in the ipc_segments_map", ptr); + return UMF_RESULT_ERROR_UNKNOWN; + } + + tracker_ipc_info_t *v = value; + + LOG_DEBUG("IPC memory region removed: tracker=%p, ptr=%p, size=%zu, " + "provider=%p, cache_entry=%p", + (void *)hTracker, ptr, v->size, v->provider, v->ipc_cache_value); + + umf_ba_free(hTracker->ipc_info_allocator, value); + + return UMF_RESULT_SUCCESS; +} + umf_memory_pool_handle_t umfMemoryTrackerGetPool(const void *ptr) { umf_alloc_info_t allocInfo = {NULL, 0, NULL}; umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); @@ -124,24 +365,78 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, return UMF_RESULT_ERROR_NOT_SUPPORTED; } - if (TRACKER->alloc_segments_map == NULL) { + if (TRACKER->alloc_segments_map[0] == NULL) { LOG_ERR("tracker's alloc_segments_map does not exist"); return UMF_RESULT_ERROR_NOT_SUPPORTED; } - uintptr_t rkey; - tracker_alloc_info_t *rvalue; - int found = critnib_find(TRACKER->alloc_segments_map, (uintptr_t)ptr, + tracker_alloc_info_t *top_most_value = NULL; + tracker_alloc_info_t *rvalue = NULL; + uintptr_t top_most_key = 0; + uintptr_t rkey = 0; + int level = 0; + int found = 0; + + do { + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + found = critnib_find(TRACKER->alloc_segments_map[level], (uintptr_t)ptr, FIND_LE, (void *)&rkey, (void **)&rvalue); + if (found && (uintptr_t)ptr < rkey + rvalue->size) { + top_most_key = rkey; + top_most_value = rvalue; + if (rvalue->n_children == 0 || + level == MAX_LEVELS_OF_ALLOC_SEGMENT_MAP - 1) { + break; + } + level++; + } + } while (found && (uintptr_t)ptr < rkey + rvalue->size && + rvalue->n_children); + + if (!top_most_value) { + LOG_DEBUG("pointer %p not found in the tracker, TRACKER=%p", ptr, + (void *)TRACKER); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + pAllocInfo->base = (void *)top_most_key; + pAllocInfo->baseSize = top_most_value->size; + pAllocInfo->pool = top_most_value->pool; + + return UMF_RESULT_SUCCESS; +} + +umf_result_t umfMemoryTrackerGetIpcInfo(const void *ptr, + umf_ipc_info_t *pIpcInfo) { + assert(pIpcInfo); + + if (ptr == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (TRACKER == NULL) { + LOG_ERR("tracker does not exist"); + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + if (TRACKER->ipc_segments_map == NULL) { + LOG_ERR("tracker's ipc_segments_map does not exist"); + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + uintptr_t rkey; + tracker_ipc_info_t *rvalue = NULL; + int found = critnib_find(TRACKER->ipc_segments_map, (uintptr_t)ptr, FIND_LE, + (void *)&rkey, (void **)&rvalue); if (!found || (uintptr_t)ptr >= rkey + rvalue->size) { LOG_DEBUG("pointer %p not found in the tracker, TRACKER=%p", ptr, (void *)TRACKER); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - pAllocInfo->base = (void *)rkey; - pAllocInfo->baseSize = rvalue->size; - pAllocInfo->pool = rvalue->pool; + pIpcInfo->base = (void *)rkey; + pIpcInfo->baseSize = rvalue->size; + pIpcInfo->provider = rvalue->provider; return UMF_RESULT_SUCCESS; } @@ -166,26 +461,38 @@ typedef struct umf_tracking_memory_provider_t { typedef struct umf_tracking_memory_provider_t umf_tracking_memory_provider_t; static umf_result_t trackingAlloc(void *hProvider, size_t size, - size_t alignment, void **ptr) { + size_t alignment, void **_ptr) { umf_tracking_memory_provider_t *p = (umf_tracking_memory_provider_t *)hProvider; umf_result_t ret = UMF_RESULT_SUCCESS; + void *ptr; assert(p->hUpstream); - ret = umfMemoryProviderAlloc(p->hUpstream, size, alignment, ptr); - if (ret != UMF_RESULT_SUCCESS || !*ptr) { + *_ptr = NULL; + + ret = umfMemoryProviderAlloc(p->hUpstream, size, alignment, &ptr); + if (ret != UMF_RESULT_SUCCESS || !ptr) { return ret; } - umf_result_t ret2 = umfMemoryTrackerAdd(p->hTracker, p->pool, *ptr, size); - if (ret2 != UMF_RESULT_SUCCESS) { + ret = umfMemoryTrackerAdd(p->hTracker, p->pool, ptr, size); + if (ret != UMF_RESULT_SUCCESS) { LOG_ERR("failed to add allocated region to the tracker, ptr = %p, size " "= %zu, ret = %d", - *ptr, size, ret2); + ptr, size, ret); + umf_result_t ret2 = umfMemoryProviderFree(p->hUpstream, ptr, size); + if (ret2 != UMF_RESULT_SUCCESS) { + LOG_ERR("upstream provider failed to free the memory: ptr = %p, " + "size = %zu, ret = %d", + ptr, size, ret2); + } + return ret; } - return ret; + *_ptr = ptr; + + return UMF_RESULT_SUCCESS; } static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, @@ -194,6 +501,8 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, umf_result_t ret = UMF_RESULT_ERROR_UNKNOWN; umf_tracking_memory_provider_t *provider = (umf_tracking_memory_provider_t *)hProvider; + tracker_alloc_info_t *parent_value = NULL; + uintptr_t parent_key = 0; tracker_alloc_info_t *splitValue = umf_ba_alloc(provider->hTracker->alloc_info_allocator); @@ -203,21 +512,27 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, splitValue->pool = provider->pool; splitValue->size = firstSize; + splitValue->n_children = 0; int r = utils_mutex_lock(&provider->hTracker->splitMergeMutex); if (r) { goto err_lock; } - tracker_alloc_info_t *value = (tracker_alloc_info_t *)critnib_get( - provider->hTracker->alloc_segments_map, (uintptr_t)ptr); + int level = 0; + + // Find the most nested (on the highest level) entry in the map + // with the `ptr` key and with no children - only such entry can be split. + tracker_alloc_info_t *value = get_most_nested_alloc_segment( + provider->hTracker, ptr, &level, &parent_key, &parent_value, + 1 /* no_children */); if (!value) { LOG_ERR("region for split is not found in the tracker"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err; } if (value->size != totalSize) { - LOG_ERR("tracked size %zu does not match requested size to split: %zu", + LOG_ERR("tracked size=%zu does not match requested size to split: %zu", value->size, totalSize); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err; @@ -230,40 +545,58 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, goto err; } + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + int cret = + critnib_insert(provider->hTracker->alloc_segments_map[level], + (uintptr_t)ptr, (void *)splitValue, 1 /* update */); + // this cannot fail since we know the element exists (nothing to allocate) + assert(cret == 0); + (void)cret; + void *highPtr = (void *)(((uintptr_t)ptr) + firstSize); size_t secondSize = totalSize - firstSize; // We'll have a duplicate entry for the range [highPtr, highValue->size] but this is fine, // the value is the same anyway and we forbid removing that range concurrently - ret = umfMemoryTrackerAdd(provider->hTracker, provider->pool, highPtr, - secondSize); + ret = umfMemoryTrackerAddAtLevel(provider->hTracker, level, provider->pool, + highPtr, secondSize, parent_key, + parent_value); if (ret != UMF_RESULT_SUCCESS) { - LOG_ERR("failed to add split region to the tracker, ptr = %p, size " - "= %zu, ret = %d", + LOG_ERR("failed to add the split region to the tracker, ptr=%p, " + "size=%zu, ret=%d", highPtr, secondSize, ret); + // revert the split + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + cret = critnib_insert(provider->hTracker->alloc_segments_map[level], + (uintptr_t)ptr, (void *)value, 1 /* update */); + // this cannot fail since we know the element exists (nothing to allocate) + assert(cret == 0); + (void)cret; // TODO: what now? should we rollback the split? This can only happen due to ENOMEM // so it's unlikely but probably the best solution would be to try to preallocate everything // (value and critnib nodes) before calling umfMemoryProviderAllocationSplit. goto err; } - int cret = - critnib_insert(provider->hTracker->alloc_segments_map, (uintptr_t)ptr, - (void *)splitValue, 1 /* update */); - // this cannot fail since we know the element exists (nothing to allocate) - assert(cret == 0); - (void)cret; - // free the original value umf_ba_free(provider->hTracker->alloc_info_allocator, value); utils_mutex_unlock(&provider->hTracker->splitMergeMutex); + LOG_DEBUG( + "split memory region (level=%i): ptr=%p, totalSize=%zu, firstSize=%zu", + level, ptr, totalSize, firstSize); + return UMF_RESULT_SUCCESS; err: utils_mutex_unlock(&provider->hTracker->splitMergeMutex); err_lock: umf_ba_free(provider->hTracker->alloc_info_allocator, splitValue); + + LOG_ERR( + "failed to split memory region: ptr=%p, totalSize=%zu, firstSize=%zu", + ptr, totalSize, firstSize); + return ret; } @@ -282,26 +615,38 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, mergedValue->pool = provider->pool; mergedValue->size = totalSize; + mergedValue->n_children = 0; + + // any different negative values + int lowLevel = -2; + int highLevel = -1; int r = utils_mutex_lock(&provider->hTracker->splitMergeMutex); if (r) { goto err_lock; } - tracker_alloc_info_t *lowValue = (tracker_alloc_info_t *)critnib_get( - provider->hTracker->alloc_segments_map, (uintptr_t)lowPtr); + tracker_alloc_info_t *lowValue = get_most_nested_alloc_segment( + provider->hTracker, lowPtr, &lowLevel, NULL, NULL, + 0 /* no_children */); // can have children if (!lowValue) { LOG_FATAL("no left value"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err_assert; } - tracker_alloc_info_t *highValue = (tracker_alloc_info_t *)critnib_get( - provider->hTracker->alloc_segments_map, (uintptr_t)highPtr); + tracker_alloc_info_t *highValue = get_most_nested_alloc_segment( + provider->hTracker, highPtr, &highLevel, NULL, NULL, + 0 /* no_children */); // can have children if (!highValue) { LOG_FATAL("no right value"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err_assert; } + if (lowLevel != highLevel) { + LOG_FATAL("tracker level mismatch"); + ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; + goto err_assert; + } if (lowValue->pool != highValue->pool) { LOG_FATAL("pool mismatch"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; @@ -313,6 +658,8 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, goto err_assert; } + mergedValue->n_children = lowValue->n_children + highValue->n_children; + ret = umfMemoryProviderAllocationMerge(provider->hUpstream, lowPtr, highPtr, totalSize); if (ret != UMF_RESULT_SUCCESS) { @@ -320,10 +667,13 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, goto not_merged; } + size_t lno = lowValue->n_children; + size_t hno = highValue->n_children; + // We'll have a duplicate entry for the range [highPtr, highValue->size] but this is fine, // the value is the same anyway and we forbid removing that range concurrently int cret = - critnib_insert(provider->hTracker->alloc_segments_map, + critnib_insert(provider->hTracker->alloc_segments_map[lowLevel], (uintptr_t)lowPtr, (void *)mergedValue, 1 /* update */); // this cannot fail since we know the element exists (nothing to allocate) assert(cret == 0); @@ -333,16 +683,23 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, umf_ba_free(provider->hTracker->alloc_info_allocator, lowValue); void *erasedhighValue = critnib_remove( - provider->hTracker->alloc_segments_map, (uintptr_t)highPtr); + provider->hTracker->alloc_segments_map[highLevel], (uintptr_t)highPtr); assert(erasedhighValue == highValue); umf_ba_free(provider->hTracker->alloc_info_allocator, erasedhighValue); utils_mutex_unlock(&provider->hTracker->splitMergeMutex); + LOG_DEBUG("merged memory regions (level=%i): lowPtr=%p (child=%zu), " + "highPtr=%p (child=%zu), totalSize=%zu", + lowLevel, lowPtr, lno, highPtr, hno, totalSize); + return UMF_RESULT_SUCCESS; err_assert: + LOG_FATAL("failed to merge memory regions: lowPtr=%p (level=%i), " + "highPtr=%p (level=%i), totalSize=%zu", + lowPtr, lowLevel, highPtr, highLevel, totalSize); assert(0); not_merged: @@ -350,6 +707,11 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, err_lock: umf_ba_free(provider->hTracker->alloc_info_allocator, mergedValue); + + LOG_ERR("failed to merge memory regions: lowPtr=%p (level=%i), highPtr=%p " + "(level=%i), totalSize=%zu", + lowPtr, lowLevel, highPtr, highLevel, totalSize); + return ret; } @@ -428,19 +790,21 @@ static umf_result_t trackingInitialize(void *params, void **ret) { #ifndef NDEBUG static void check_if_tracker_is_empty(umf_memory_tracker_handle_t hTracker, umf_memory_pool_handle_t pool) { - uintptr_t rkey; - void *rvalue; size_t n_items = 0; - uintptr_t last_key = 0; - while (1 == critnib_find((critnib *)hTracker->alloc_segments_map, last_key, - FIND_G, &rkey, &rvalue)) { - tracker_alloc_info_t *value = (tracker_alloc_info_t *)rvalue; - if (value->pool == pool || pool == NULL) { - n_items++; - } + for (int i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { + uintptr_t last_key = 0; + uintptr_t rkey; + tracker_alloc_info_t *rvalue; + + while (1 == critnib_find(hTracker->alloc_segments_map[i], last_key, + FIND_G, &rkey, (void **)&rvalue)) { + if (rvalue->pool == pool || pool == NULL) { + n_items++; + } - last_key = rkey; + last_key = rkey; + } } if (n_items) { @@ -618,17 +982,17 @@ ipcOpenedCacheEvictionCallback(const ipc_opened_cache_key_t *key, const ipc_opened_cache_value_t *value) { umf_tracking_memory_provider_t *p = (umf_tracking_memory_provider_t *)key->local_provider; - // umfMemoryTrackerRemove should be called before umfMemoryProviderCloseIPCHandle + // umfMemoryTrackerRemoveIpcSegment should be called before umfMemoryProviderCloseIPCHandle // to avoid a race condition. If the order would be different, other thread - // could allocate the memory at address `ptr` before a call to umfMemoryTrackerRemove + // could allocate the memory at address `ptr` before a call to umfMemoryTrackerRemoveIpcSegment // resulting in inconsistent state. if (value->mapped_base_ptr) { - umf_result_t ret = - umfMemoryTrackerRemove(p->hTracker, value->mapped_base_ptr); + umf_result_t ret = umfMemoryTrackerRemoveIpcSegment( + p->hTracker, value->mapped_base_ptr); if (ret != UMF_RESULT_SUCCESS) { // DO NOT return an error here, because the tracking provider // cannot change behaviour of the upstream provider. - LOG_ERR("failed to remove the region from the tracker, ptr=%p, " + LOG_ERR("failed to remove the region from the IPC tracker, ptr=%p, " "size=%zu, ret = %d", value->mapped_base_ptr, value->mapped_size, ret); } @@ -641,12 +1005,13 @@ ipcOpenedCacheEvictionCallback(const ipc_opened_cache_key_t *key, } } -static umf_result_t upstreamOpenIPCHandle(umf_tracking_memory_provider_t *p, - void *providerIpcData, - size_t bufferSize, void **ptr) { +static umf_result_t +upstreamOpenIPCHandle(umf_tracking_memory_provider_t *p, void *providerIpcData, + size_t bufferSize, + ipc_opened_cache_value_t *cache_entry) { void *mapped_ptr = NULL; assert(p != NULL); - assert(ptr != NULL); + assert(cache_entry != NULL); umf_result_t ret = umfMemoryProviderOpenIPCHandle( p->hUpstream, providerIpcData, &mapped_ptr); if (ret != UMF_RESULT_SUCCESS) { @@ -655,7 +1020,21 @@ static umf_result_t upstreamOpenIPCHandle(umf_tracking_memory_provider_t *p, } assert(mapped_ptr != NULL); - ret = umfMemoryTrackerAdd(p->hTracker, p->pool, mapped_ptr, bufferSize); + // Today umfMemoryTrackerAddIpcSegment requires the memory provider handle + // to know which tracking provider instance opened the IPC handle. + // The `p` points to the tracking provider private data. + // Because of that we get handle to the tracking provider instance + // using `p->pool->provider`. + // + // TODO: + // Today we always create a pool and get an IPC handler from the pool. + // And tracking provider is always created together with a pool. + // And the IPC handler is a tracking memory provider in fact. + // However, we are considering adding an API that allows IPC handler creation + // from scratch (without creating a memory pool). In that case, we will + // create a tracker provider without a pool. So p->pool might be NULL in the future. + ret = umfMemoryTrackerAddIpcSegment(p->hTracker, mapped_ptr, bufferSize, + p->pool->provider, cache_entry); if (ret != UMF_RESULT_SUCCESS) { LOG_ERR("failed to add IPC region to the tracker, ptr=%p, " "size=%zu, " @@ -670,7 +1049,8 @@ static umf_result_t upstreamOpenIPCHandle(umf_tracking_memory_provider_t *p, return ret; } - *ptr = mapped_ptr; + cache_entry->mapped_size = bufferSize; + utils_atomic_store_release_ptr(&(cache_entry->mapped_base_ptr), mapped_ptr); return UMF_RESULT_SUCCESS; } @@ -705,45 +1085,46 @@ static umf_result_t trackingOpenIpcHandle(void *provider, void *providerIpcData, void *mapped_ptr = NULL; utils_atomic_load_acquire_ptr(&(cache_entry->mapped_base_ptr), (void **)&mapped_ptr); - if (mapped_ptr == NULL) { + if (mapped_ptr == NULL) { // new cache entry utils_mutex_lock(&(cache_entry->mmap_lock)); utils_atomic_load_acquire_ptr(&(cache_entry->mapped_base_ptr), (void **)&mapped_ptr); if (mapped_ptr == NULL) { ret = upstreamOpenIPCHandle(p, providerIpcData, - ipcUmfData->baseSize, &mapped_ptr); - if (ret == UMF_RESULT_SUCCESS) { - // Put to the cache - cache_entry->mapped_size = ipcUmfData->baseSize; - utils_atomic_store_release_ptr(&(cache_entry->mapped_base_ptr), - mapped_ptr); - } + ipcUmfData->baseSize, cache_entry); } + mapped_ptr = cache_entry->mapped_base_ptr; utils_mutex_unlock(&(cache_entry->mmap_lock)); } if (ret == UMF_RESULT_SUCCESS) { + assert(mapped_ptr != NULL); *ptr = mapped_ptr; } return ret; } +static tracker_ipc_info_t *getTrackerIpcInfo(const void *ptr) { + assert(ptr); + + uintptr_t key = (uintptr_t)ptr; + tracker_ipc_info_t *value = critnib_get(TRACKER->ipc_segments_map, key); + + return value; +} + static umf_result_t trackingCloseIpcHandle(void *provider, void *ptr, size_t size) { (void)provider; - (void)ptr; - (void)size; - // We keep opened IPC handles in the p->hIpcMappedCache. - // IPC handle is closed when it is evicted from the cache - // or when cache is destroyed. - // - // TODO: today the size of the IPC cache is infinite. - // When the threshold for the cache size is implemented - // we need to introduce a reference counting mechanism. - // The trackingOpenIpcHandle will increment the refcount for the corresponding entry. - // The trackingCloseIpcHandle will decrement the refcount for the corresponding cache entry. - return UMF_RESULT_SUCCESS; + tracker_ipc_info_t *trackerIpcInfo = getTrackerIpcInfo(ptr); + + if (!trackerIpcInfo) { + LOG_ERR("failed to get tracker ipc info, ptr=%p, size=%zu", ptr, size); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + return umfIpcHandleMappedCacheRelease(trackerIpcInfo->ipc_cache_value); } umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { @@ -813,6 +1194,8 @@ umf_memory_tracker_handle_t umfMemoryTrackerCreate(void) { return NULL; } + memset(handle, 0, sizeof(struct umf_memory_tracker_t)); + umf_ba_pool_t *alloc_info_allocator = umf_ba_create(sizeof(struct tracker_alloc_info_t)); if (!alloc_info_allocator) { @@ -826,9 +1209,23 @@ umf_memory_tracker_handle_t umfMemoryTrackerCreate(void) { goto err_destroy_alloc_info_allocator; } - handle->alloc_segments_map = critnib_new(); - if (!handle->alloc_segments_map) { - goto err_destroy_mutex; + int i; + for (i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { + handle->alloc_segments_map[i] = critnib_new(); + if (!handle->alloc_segments_map[i]) { + goto err_destroy_alloc_segments_map; + } + } + + handle->ipc_info_allocator = + umf_ba_create(sizeof(struct tracker_ipc_info_t)); + if (!handle->ipc_info_allocator) { + goto err_destroy_alloc_segments_map; + } + + handle->ipc_segments_map = critnib_new(); + if (!handle->ipc_segments_map) { + goto err_destroy_ipc_info_allocator; } LOG_DEBUG("tracker created, handle=%p, alloc_segments_map=%p", @@ -836,7 +1233,14 @@ umf_memory_tracker_handle_t umfMemoryTrackerCreate(void) { return handle; -err_destroy_mutex: +err_destroy_ipc_info_allocator: + umf_ba_destroy(handle->ipc_info_allocator); +err_destroy_alloc_segments_map: + for (i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { + if (handle->alloc_segments_map[i]) { + critnib_delete(handle->alloc_segments_map[i]); + } + } utils_mutex_destroy_not_free(&handle->splitMergeMutex); err_destroy_alloc_info_allocator: umf_ba_destroy(alloc_info_allocator); @@ -864,10 +1268,18 @@ void umfMemoryTrackerDestroy(umf_memory_tracker_handle_t handle) { // We have to zero all inner pointers, // because the tracker handle can be copied // and used in many places. - critnib_delete(handle->alloc_segments_map); - handle->alloc_segments_map = NULL; + for (int i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { + if (handle->alloc_segments_map[i]) { + critnib_delete(handle->alloc_segments_map[i]); + handle->alloc_segments_map[i] = NULL; + } + } utils_mutex_destroy_not_free(&handle->splitMergeMutex); umf_ba_destroy(handle->alloc_info_allocator); handle->alloc_info_allocator = NULL; + critnib_delete(handle->ipc_segments_map); + handle->ipc_segments_map = NULL; + umf_ba_destroy(handle->ipc_info_allocator); + handle->ipc_info_allocator = NULL; umf_ba_global_free(handle); } diff --git a/src/provider/provider_tracking.h b/src/provider/provider_tracking.h index 9e868cf311..842449be5c 100644 --- a/src/provider/provider_tracking.h +++ b/src/provider/provider_tracking.h @@ -45,6 +45,15 @@ typedef struct umf_alloc_info_t { umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, umf_alloc_info_t *pAllocInfo); +typedef struct umf_ipc_info_t { + void *base; + size_t baseSize; + umf_memory_provider_handle_t provider; +} umf_ipc_info_t; + +umf_result_t umfMemoryTrackerGetIpcInfo(const void *ptr, + umf_ipc_info_t *pIpcInfo); + // Creates a memory provider that tracks each allocation/deallocation through umf_memory_tracker_handle_t and // forwards all requests to hUpstream memory Provider. hUpstream lifetime should be managed by the user of this function. umf_result_t umfTrackingMemoryProviderCreate( diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index a0bff39fd8..976a2cb626 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (C) 2023-2024 Intel Corporation +# Copyright (C) 2023-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -7,15 +7,14 @@ include(FindThreads) set(UMF_UTILS_SOURCES_COMMON utils_common.c utils_log.c utils_load_library.c) -set(UMF_UTILS_SOURCES_POSIX utils_posix_common.c utils_posix_concurrency.c - utils_posix_math.c) +set(UMF_UTILS_SOURCES_POSIX utils_posix_common.c utils_posix_concurrency.c) set(UMF_UTILS_SOURCES_LINUX utils_linux_common.c) set(UMF_UTILS_SOURCES_MACOSX utils_macosx_common.c) set(UMF_UTILS_SOURCES_WINDOWS utils_windows_common.c - utils_windows_concurrency.c utils_windows_math.c) + utils_windows_concurrency.c) if(UMF_USE_VALGRIND) if(UMF_USE_ASAN diff --git a/src/utils/utils_concurrency.h b/src/utils/utils_concurrency.h index 638c1c4262..81e0bbc712 100644 --- a/src/utils/utils_concurrency.h +++ b/src/utils/utils_concurrency.h @@ -89,18 +89,6 @@ void utils_init_once(UTIL_ONCE_FLAG *flag, void (*onceCb)(void)); #if defined(_WIN32) -static inline unsigned char utils_lssb_index(long long value) { - unsigned long ret; - _BitScanForward64(&ret, value); - return (unsigned char)ret; -} - -static inline unsigned char utils_mssb_index(long long value) { - unsigned long ret; - _BitScanReverse64(&ret, value); - return (unsigned char)ret; -} - // There is no good way to do atomic_load on windows... static inline void utils_atomic_load_acquire_u64(uint64_t *ptr, uint64_t *out) { // NOTE: Windows cl complains about direct accessing 'ptr' which is next @@ -168,10 +156,18 @@ static inline bool utils_compare_exchange_u64(uint64_t *ptr, uint64_t *expected, return false; } -#else // !defined(_WIN32) +static inline void utils_atomic_store_release_u64(void *ptr, uint64_t val) { + ASSERT_IS_ALIGNED((uintptr_t)ptr, 8); + LONG64 out; + LONG64 desired = (LONG64)val; + LONG64 expected = 0; + while (expected != (out = InterlockedCompareExchange64( + (LONG64 volatile *)ptr, desired, expected))) { + expected = out; + } +} -#define utils_lssb_index(x) ((unsigned char)__builtin_ctzll(x)) -#define utils_mssb_index(x) ((unsigned char)(63 - __builtin_clzll(x))) +#else // !defined(_WIN32) static inline void utils_atomic_load_acquire_u64(uint64_t *ptr, uint64_t *out) { ASSERT_IS_ALIGNED((uintptr_t)ptr, 8); @@ -187,10 +183,9 @@ static inline void utils_atomic_load_acquire_ptr(void **ptr, void **out) { utils_annotate_acquire(ptr); } -static inline void utils_atomic_store_release_u64(uint64_t *ptr, uint64_t val) { +static inline void utils_atomic_store_release_u64(void *ptr, uint64_t val) { ASSERT_IS_ALIGNED((uintptr_t)ptr, 8); - utils_annotate_release(ptr); - __atomic_store_n(ptr, val, memory_order_release); + __atomic_store_n((uintptr_t *)ptr, (uintptr_t)val, memory_order_release); } static inline void utils_atomic_store_release_ptr(void **ptr, void *val) { diff --git a/src/utils/utils_math.h b/src/utils/utils_math.h index c78be11367..0e58fc38df 100644 --- a/src/utils/utils_math.h +++ b/src/utils/utils_math.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023-2024 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -11,16 +11,58 @@ #define UMF_MATH_H 1 #include +#include #include +#include #ifdef __cplusplus extern "C" { #endif -size_t getLeftmostSetBitPos(size_t num); +#if defined(_WIN32) -// Logarithm is an index of the most significant non-zero bit. -static inline size_t log2Utils(size_t num) { return getLeftmostSetBitPos(num); } +#include "utils_windows_intrin.h" + +#pragma intrinsic(_BitScanReverse64) +#pragma intrinsic(_BitScanForward64) + +// Retrieves the position of the leftmost set bit. +// The position of the bit is counted from 0 +// e.g. for 01000011110 the position equals 9. +static inline size_t utils_msb64(uint64_t num) { + assert(num != 0 && + "Finding leftmost set bit when number equals zero is undefined"); + unsigned long index = 0; + _BitScanReverse64(&index, num); + return (size_t)index; +} + +static inline size_t utils_lsb64(uint64_t num) { + assert(num != 0 && + "Finding rightmost set bit when number equals zero is undefined"); + unsigned long index = 0; + _BitScanForward64(&index, num); + return (size_t)index; +} + +#else // !defined(_WIN32) + +// Retrieves the position of the leftmost set bit. +// The position of the bit is counted from 0 +// e.g. for 01000011110 the position equals 9. +static inline size_t utils_msb64(uint64_t num) { + assert(num != 0 && + "Finding leftmost set bit when number equals zero is undefined"); + return 63 - __builtin_clzll(num); +} + +static inline size_t utils_lsb64(uint64_t num) { + assert(num != 0 && + "Finding rightmost set bit when number equals zero is undefined"); + return __builtin_ctzll(num); +} + +#endif // !defined(_WIN32) #ifdef __cplusplus } diff --git a/src/utils/utils_posix_math.c b/src/utils/utils_posix_math.c deleted file mode 100644 index 465b687725..0000000000 --- a/src/utils/utils_posix_math.c +++ /dev/null @@ -1,20 +0,0 @@ -/* - * - * Copyright (C) 2023 Intel Corporation - * - * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - * - */ - -#include "utils_math.h" -#include - -// Retrieves the position of the leftmost set bit. -// The position of the bit is counted from 0 -// e.g. for 01000011110 the position equals 9. -size_t getLeftmostSetBitPos(size_t num) { - assert(num != 0 && - "Finding leftmost set bit when number equals zero is undefined"); - return (sizeof(num) * CHAR_BIT - 1) - __builtin_clzll(num); -} diff --git a/src/utils/utils_windows_math.c b/src/utils/utils_windows_math.c deleted file mode 100644 index cd21ae696f..0000000000 --- a/src/utils/utils_windows_math.c +++ /dev/null @@ -1,24 +0,0 @@ -/* - * - * Copyright (C) 2023-2025 Intel Corporation - * - * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - * - */ - -#include "utils_math.h" -#include "utils_windows_intrin.h" - -#pragma intrinsic(_BitScanReverse) - -// Retrieves the position of the leftmost set bit. -// The position of the bit is counted from 0 -// e.g. for 01000011110 the position equals 9. -size_t getLeftmostSetBitPos(size_t num) { - assert(num != 0 && - "Finding leftmost set bit when number equals zero is undefined"); - unsigned long index = 0; - _BitScanReverse(&index, (unsigned long)num); - return (size_t)index; -} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5da6f54665..08498abf8f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -116,8 +116,9 @@ function(add_umf_test) # * NAME - a name of the test # * SRCS - source files # * LIBS - libraries to be linked with + # * ENVS - environment variables set(oneValueArgs NAME) - set(multiValueArgs SRCS LIBS) + set(multiValueArgs SRCS LIBS ENVS) cmake_parse_arguments( ARG "" @@ -139,6 +140,9 @@ function(add_umf_test) WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_tests_properties(${TEST_NAME} PROPERTIES LABELS "umf") + if(ARG_ENVS) + set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT ${ARG_ENVS}) + endif() if(WINDOWS) # add PATH to DLL on Windows @@ -349,6 +353,15 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented NAME provider_fixed_memory SRCS provider_fixed_memory.cpp LIBS ${UMF_UTILS_FOR_TEST}) + add_umf_test( + NAME provider_tracking + SRCS provider_tracking.cpp + LIBS ${UMF_UTILS_FOR_TEST}) + add_umf_test( + NAME provider_tracking_fixture_tests + SRCS provider_tracking_fixture_tests.cpp malloc_compliance_tests.cpp + ${BA_SOURCES_FOR_TEST} + LIBS ${UMF_UTILS_FOR_TEST}) # This test requires Linux-only file memory provider if(UMF_POOL_JEMALLOC_ENABLED) @@ -521,6 +534,12 @@ add_umf_test( SRCS ipcAPI.cpp ${BA_SOURCES_FOR_TEST} LIBS ${UMF_UTILS_FOR_TEST}) +add_umf_test( + NAME ipc_max_opened_limit + SRCS ipcAPI.cpp ${BA_SOURCES_FOR_TEST} + LIBS ${UMF_UTILS_FOR_TEST} + ENVS "UMF_MAX_OPENED_IPC_HANDLES=10") + add_umf_test(NAME ipc_negative SRCS ipc_negative.cpp) function(add_umf_ipc_test) diff --git a/test/c_api/test_ut_asserts.h b/test/c_api/test_ut_asserts.h index 834d39bda8..b73f0cd19e 100644 --- a/test/c_api/test_ut_asserts.h +++ b/test/c_api/test_ut_asserts.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023-2024 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -9,7 +9,7 @@ /* The project uses GTEST framework for testing, which is not supported in C - These asserts should NOT be used in other purposes than for testing C API + These asserts should NOT be used in other purposes than for testing C API */ #ifndef UMF_TEST_UT_ASSERTS_H diff --git a/test/coarse_lib.cpp b/test/coarse_lib.cpp index c2e1f9c856..7611833899 100644 --- a/test/coarse_lib.cpp +++ b/test/coarse_lib.cpp @@ -99,7 +99,7 @@ static void coarse_params_set_default(coarse_params_t *coarse_params, } umf_memory_provider_ops_t UMF_MALLOC_MEMORY_PROVIDER_OPS = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); struct CoarseWithMemoryStrategyTest : umf_test::test, diff --git a/test/common/pool.hpp b/test/common/pool.hpp index a5b4afc15b..558b9d665a 100644 --- a/test/common/pool.hpp +++ b/test/common/pool.hpp @@ -22,8 +22,8 @@ #include #include "base.hpp" -#include "cpp_helpers.hpp" #include "provider.hpp" +#include "utils/cpp_helpers.hpp" namespace umf_test { @@ -38,7 +38,7 @@ createPoolChecked(umf_memory_pool_ops_t *ops, } auto wrapPoolUnique(umf_memory_pool_handle_t hPool) { - return umf::pool_unique_handle_t(hPool, &umfPoolDestroy); + return umf_test::pool_unique_handle_t(hPool, &umfPoolDestroy); } bool isReallocSupported(umf_memory_pool_handle_t hPool) { @@ -149,7 +149,7 @@ struct malloc_pool : public pool_base_t { }; umf_memory_pool_ops_t MALLOC_POOL_OPS = - umf::poolMakeCOps(); + umf_test::poolMakeCOps(); static constexpr size_t DEFAULT_DISJOINT_SLAB_MIN_SIZE = 4096; static constexpr size_t DEFAULT_DISJOINT_MAX_POOLABLE_SIZE = 4096; diff --git a/test/common/provider.hpp b/test/common/provider.hpp index 148f34dc89..38fe7336ec 100644 --- a/test/common/provider.hpp +++ b/test/common/provider.hpp @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023-2024 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -15,8 +15,8 @@ #include "base.hpp" #include "base_alloc_global.h" -#include "cpp_helpers.hpp" #include "test_helpers.h" +#include "utils/cpp_helpers.hpp" namespace umf_test { @@ -29,7 +29,8 @@ createProviderChecked(umf_memory_provider_ops_t *ops, void *params) { } auto wrapProviderUnique(umf_memory_provider_handle_t hProvider) { - return umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); + return umf_test::provider_unique_handle_t(hProvider, + &umfMemoryProviderDestroy); } typedef struct provider_base_t { @@ -97,7 +98,7 @@ typedef struct provider_base_t { } provider_base_t; umf_memory_provider_ops_t BASE_PROVIDER_OPS = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); struct provider_ba_global : public provider_base_t { umf_result_t alloc(size_t size, size_t align, void **ptr) noexcept { @@ -127,7 +128,7 @@ struct provider_ba_global : public provider_base_t { }; umf_memory_provider_ops_t BA_GLOBAL_PROVIDER_OPS = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); struct provider_mock_out_of_mem : public provider_base_t { provider_ba_global helper_prov; @@ -152,7 +153,7 @@ struct provider_mock_out_of_mem : public provider_base_t { }; umf_memory_provider_ops_t MOCK_OUT_OF_MEM_PROVIDER_OPS = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); } // namespace umf_test diff --git a/test/disjoint_pool_file_prov.cpp b/test/disjoint_pool_file_prov.cpp index b874d2a49b..58e15f5714 100644 --- a/test/disjoint_pool_file_prov.cpp +++ b/test/disjoint_pool_file_prov.cpp @@ -20,7 +20,7 @@ using umf_test::test; #define FILE_PATH ((char *)"tmp_file") umf_memory_provider_ops_t UMF_MALLOC_MEMORY_PROVIDER_OPS = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); struct FileWithMemoryStrategyTest : umf_test::test, diff --git a/test/ipcAPI.cpp b/test/ipcAPI.cpp index 4298963088..c0642dd760 100644 --- a/test/ipcAPI.cpp +++ b/test/ipcAPI.cpp @@ -109,7 +109,7 @@ provider_mock_ipc::allocations_mutex_type provider_mock_ipc::alloc_mutex; provider_mock_ipc::allocations_map_type provider_mock_ipc::allocations; static umf_memory_provider_ops_t IPC_MOCK_PROVIDER_OPS = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); HostMemoryAccessor hostMemoryAccessor; diff --git a/test/ipcFixtures.hpp b/test/ipcFixtures.hpp index 57bd040792..cf31ff7584 100644 --- a/test/ipcFixtures.hpp +++ b/test/ipcFixtures.hpp @@ -68,6 +68,18 @@ using ipcTestParams = struct umfIpcTest : umf_test::test, ::testing::WithParamInterface { umfIpcTest() {} + size_t getOpenedIpcCacheSize() { + const char *max_size_str = getenv("UMF_MAX_OPENED_IPC_HANDLES"); + if (max_size_str) { + char *endptr; + size_t max_size = strtoul(max_size_str, &endptr, 10); + EXPECT_EQ(*endptr, '\0'); + if (*endptr == '\0') { + return max_size; + } + } + return 0; + } void SetUp() override { test::SetUp(); auto [pool_ops, pool_params_create, pool_params_destroy, provider_ops, @@ -80,11 +92,12 @@ struct umfIpcTest : umf_test::test, providerParamsCreate = provider_params_create; providerParamsDestroy = provider_params_destroy; memAccessor = accessor; + openedIpcCacheSize = getOpenedIpcCacheSize(); } void TearDown() override { test::TearDown(); } - umf::pool_unique_handle_t makePool() { + umf_test::pool_unique_handle_t makePool() { // TODO: The function is similar to poolCreateExt function // from memoryPool.hpp umf_memory_provider_handle_t hProvider = NULL; @@ -134,7 +147,7 @@ struct umfIpcTest : umf_test::test, poolParamsDestroy(poolParams); } - return umf::pool_unique_handle_t(hPool, &umfPoolDestroy); + return umf_test::pool_unique_handle_t(hPool, &umfPoolDestroy); } struct stats_type { @@ -160,12 +173,13 @@ struct umfIpcTest : umf_test::test, umf_memory_provider_ops_t *providerOps = nullptr; pfnProviderParamsCreate providerParamsCreate = nullptr; pfnProviderParamsDestroy providerParamsDestroy = nullptr; + size_t openedIpcCacheSize = 0; void concurrentGetConcurrentPutHandles(bool shuffle) { std::vector ptrs; constexpr size_t ALLOC_SIZE = 100; constexpr size_t NUM_POINTERS = 100; - umf::pool_unique_handle_t pool = makePool(); + umf_test::pool_unique_handle_t pool = makePool(); ASSERT_NE(pool.get(), nullptr); for (size_t i = 0; i < NUM_POINTERS; ++i) { @@ -223,7 +237,7 @@ struct umfIpcTest : umf_test::test, std::vector ptrs; constexpr size_t ALLOC_SIZE = 100; constexpr size_t NUM_POINTERS = 100; - umf::pool_unique_handle_t pool = makePool(); + umf_test::pool_unique_handle_t pool = makePool(); ASSERT_NE(pool.get(), nullptr); for (size_t i = 0; i < NUM_POINTERS; ++i) { @@ -264,11 +278,161 @@ struct umfIpcTest : umf_test::test, pool.reset(nullptr); EXPECT_EQ(stat.putCount, stat.getCount); } + + void concurrentOpenConcurrentCloseHandles(bool shuffle) { + umf_result_t ret; + std::vector ptrs; + constexpr size_t ALLOC_SIZE = 100; + constexpr size_t NUM_POINTERS = 100; + umf_test::pool_unique_handle_t pool = makePool(); + ASSERT_NE(pool.get(), nullptr); + + for (size_t i = 0; i < NUM_POINTERS; ++i) { + void *ptr = umfPoolMalloc(pool.get(), ALLOC_SIZE); + EXPECT_NE(ptr, nullptr); + ptrs.push_back(ptr); + } + + std::vector ipcHandles; + for (size_t i = 0; i < NUM_POINTERS; ++i) { + umf_ipc_handle_t ipcHandle; + size_t handleSize; + ret = umfGetIPCHandle(ptrs[i], &ipcHandle, &handleSize); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ipcHandles.push_back(ipcHandle); + } + + std::array, NTHREADS> openedIpcHandles; + umf_ipc_handler_handle_t ipcHandler = nullptr; + ret = umfPoolGetIPCHandler(pool.get(), &ipcHandler); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(ipcHandler, nullptr); + + umf_test::syncthreads_barrier syncthreads(NTHREADS); + + auto openHandlesFn = [shuffle, &ipcHandles, &openedIpcHandles, + &syncthreads, ipcHandler](size_t tid) { + // Each thread gets a copy of the pointers to shuffle them + std::vector localIpcHandles = ipcHandles; + if (shuffle) { + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(localIpcHandles.begin(), localIpcHandles.end(), g); + } + syncthreads(); + for (auto ipcHandle : localIpcHandles) { + void *ptr; + umf_result_t ret = + umfOpenIPCHandle(ipcHandler, ipcHandle, &ptr); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + openedIpcHandles[tid].push_back(ptr); + } + }; + + umf_test::parallel_exec(NTHREADS, openHandlesFn); + + auto closeHandlesFn = [&openedIpcHandles, &syncthreads](size_t tid) { + syncthreads(); + for (void *ptr : openedIpcHandles[tid]) { + umf_result_t ret = umfCloseIPCHandle(ptr); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + } + }; + + umf_test::parallel_exec(NTHREADS, closeHandlesFn); + + for (auto ipcHandle : ipcHandles) { + ret = umfPutIPCHandle(ipcHandle); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + } + + for (void *ptr : ptrs) { + ret = umfPoolFree(pool.get(), ptr); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + } + + pool.reset(nullptr); + EXPECT_EQ(stat.getCount, stat.allocCount); + EXPECT_EQ(stat.putCount, stat.getCount); + EXPECT_EQ(stat.openCount, stat.allocCount); + EXPECT_EQ(stat.openCount, stat.closeCount); + } + + void concurrentOpenCloseHandles(bool shuffle) { + umf_result_t ret; + std::vector ptrs; + constexpr size_t ALLOC_SIZE = 100; + constexpr size_t NUM_POINTERS = 100; + umf_test::pool_unique_handle_t pool = makePool(); + ASSERT_NE(pool.get(), nullptr); + + for (size_t i = 0; i < NUM_POINTERS; ++i) { + void *ptr = umfPoolMalloc(pool.get(), ALLOC_SIZE); + EXPECT_NE(ptr, nullptr); + ptrs.push_back(ptr); + } + + std::vector ipcHandles; + for (size_t i = 0; i < NUM_POINTERS; ++i) { + umf_ipc_handle_t ipcHandle; + size_t handleSize; + ret = umfGetIPCHandle(ptrs[i], &ipcHandle, &handleSize); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ipcHandles.push_back(ipcHandle); + } + + umf_ipc_handler_handle_t ipcHandler = nullptr; + ret = umfPoolGetIPCHandler(pool.get(), &ipcHandler); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(ipcHandler, nullptr); + + umf_test::syncthreads_barrier syncthreads(NTHREADS); + + auto openCloseHandlesFn = [shuffle, &ipcHandles, &syncthreads, + ipcHandler](size_t) { + // Each thread gets a copy of the pointers to shuffle them + std::vector localIpcHandles = ipcHandles; + if (shuffle) { + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(localIpcHandles.begin(), localIpcHandles.end(), g); + } + syncthreads(); + for (auto ipcHandle : localIpcHandles) { + void *ptr; + umf_result_t ret = + umfOpenIPCHandle(ipcHandler, ipcHandle, &ptr); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ret = umfCloseIPCHandle(ptr); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + } + }; + + umf_test::parallel_exec(NTHREADS, openCloseHandlesFn); + + for (auto ipcHandle : ipcHandles) { + ret = umfPutIPCHandle(ipcHandle); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + } + + for (void *ptr : ptrs) { + ret = umfPoolFree(pool.get(), ptr); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + } + + pool.reset(nullptr); + EXPECT_EQ(stat.getCount, stat.allocCount); + EXPECT_EQ(stat.putCount, stat.getCount); + if (openedIpcCacheSize == 0) { + EXPECT_EQ(stat.openCount, stat.allocCount); + } + EXPECT_EQ(stat.openCount, stat.closeCount); + } }; TEST_P(umfIpcTest, GetIPCHandleSize) { size_t size = 0; - umf::pool_unique_handle_t pool = makePool(); + umf_test::pool_unique_handle_t pool = makePool(); ASSERT_NE(pool.get(), nullptr); umf_result_t ret = umfPoolGetIPCHandleSize(pool.get(), &size); @@ -281,7 +445,7 @@ TEST_P(umfIpcTest, GetIPCHandleSizeInvalidArgs) { umf_result_t ret = umfPoolGetIPCHandleSize(nullptr, &size); EXPECT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); - umf::pool_unique_handle_t pool = makePool(); + umf_test::pool_unique_handle_t pool = makePool(); ASSERT_NE(pool.get(), nullptr); ret = umfPoolGetIPCHandleSize(pool.get(), nullptr); @@ -299,7 +463,7 @@ TEST_P(umfIpcTest, GetIPCHandleInvalidArgs) { ret = umfGetIPCHandle(ptr, &ipcHandle, &handleSize); EXPECT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); - umf::pool_unique_handle_t pool = makePool(); + umf_test::pool_unique_handle_t pool = makePool(); ASSERT_NE(pool.get(), nullptr); ptr = umfPoolMalloc(pool.get(), SIZE); @@ -324,7 +488,7 @@ TEST_P(umfIpcTest, CloseIPCHandleInvalidPtr) { TEST_P(umfIpcTest, BasicFlow) { constexpr size_t SIZE = 100; std::vector expected_data(SIZE); - umf::pool_unique_handle_t pool = makePool(); + umf_test::pool_unique_handle_t pool = makePool(); ASSERT_NE(pool.get(), nullptr); int *ptr = (int *)umfPoolMalloc(pool.get(), SIZE * sizeof(int)); @@ -389,73 +553,9 @@ TEST_P(umfIpcTest, BasicFlow) { EXPECT_EQ(stat.closeCount, stat.openCount); } -TEST_P(umfIpcTest, GetPoolByOpenedHandle) { - constexpr size_t SIZE = 100; - constexpr size_t NUM_ALLOCS = 100; - constexpr size_t NUM_POOLS = 4; - void *ptrs[NUM_ALLOCS]; - void *openedPtrs[NUM_POOLS][NUM_ALLOCS]; - std::vector pools_to_open; - umf::pool_unique_handle_t pool = makePool(); - ASSERT_NE(pool.get(), nullptr); - - for (size_t i = 0; i < NUM_POOLS; ++i) { - pools_to_open.push_back(makePool()); - } - - for (size_t i = 0; i < NUM_ALLOCS; ++i) { - void *ptr = umfPoolMalloc(pool.get(), SIZE); - ASSERT_NE(ptr, nullptr); - ptrs[i] = ptr; - } - - for (size_t i = 0; i < NUM_ALLOCS; ++i) { - umf_ipc_handle_t ipcHandle = nullptr; - size_t handleSize = 0; - umf_result_t ret = umfGetIPCHandle(ptrs[i], &ipcHandle, &handleSize); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - - for (size_t pool_id = 0; pool_id < NUM_POOLS; pool_id++) { - void *ptr = nullptr; - umf_ipc_handler_handle_t ipcHandler = nullptr; - ret = - umfPoolGetIPCHandler(pools_to_open[pool_id].get(), &ipcHandler); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - ASSERT_NE(ipcHandler, nullptr); - - ret = umfOpenIPCHandle(ipcHandler, ipcHandle, &ptr); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - openedPtrs[pool_id][i] = ptr; - } - - ret = umfPutIPCHandle(ipcHandle); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - } - - for (size_t pool_id = 0; pool_id < NUM_POOLS; pool_id++) { - for (size_t i = 0; i < NUM_ALLOCS; ++i) { - umf_memory_pool_handle_t openedPool = - umfPoolByPtr(openedPtrs[pool_id][i]); - EXPECT_EQ(openedPool, pools_to_open[pool_id].get()); - } - } - - for (size_t pool_id = 0; pool_id < NUM_POOLS; pool_id++) { - for (size_t i = 0; i < NUM_ALLOCS; ++i) { - umf_result_t ret = umfCloseIPCHandle(openedPtrs[pool_id][i]); - EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - } - } - - for (size_t i = 0; i < NUM_ALLOCS; ++i) { - umf_result_t ret = umfFree(ptrs[i]); - EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - } -} - TEST_P(umfIpcTest, AllocFreeAllocTest) { constexpr size_t SIZE = 64 * 1024; - umf::pool_unique_handle_t pool = makePool(); + umf_test::pool_unique_handle_t pool = makePool(); ASSERT_NE(pool.get(), nullptr); umf_ipc_handler_handle_t ipcHandler = nullptr; @@ -516,9 +616,9 @@ TEST_P(umfIpcTest, AllocFreeAllocTest) { TEST_P(umfIpcTest, openInTwoIpcHandlers) { constexpr size_t SIZE = 100; std::vector expected_data(SIZE); - umf::pool_unique_handle_t pool1 = makePool(); + umf_test::pool_unique_handle_t pool1 = makePool(); ASSERT_NE(pool1.get(), nullptr); - umf::pool_unique_handle_t pool2 = makePool(); + umf_test::pool_unique_handle_t pool2 = makePool(); ASSERT_NE(pool2.get(), nullptr); umf_ipc_handler_handle_t ipcHandler1 = nullptr; umf_ipc_handler_handle_t ipcHandler2 = nullptr; @@ -593,75 +693,20 @@ TEST_P(umfIpcTest, ConcurrentGetPutHandlesShuffled) { concurrentGetPutHandles(true); } -TEST_P(umfIpcTest, ConcurrentOpenCloseHandles) { - umf_result_t ret; - std::vector ptrs; - constexpr size_t ALLOC_SIZE = 100; - constexpr size_t NUM_POINTERS = 100; - umf::pool_unique_handle_t pool = makePool(); - ASSERT_NE(pool.get(), nullptr); - - for (size_t i = 0; i < NUM_POINTERS; ++i) { - void *ptr = umfPoolMalloc(pool.get(), ALLOC_SIZE); - EXPECT_NE(ptr, nullptr); - ptrs.push_back(ptr); - } - - std::array ipcHandles; - for (size_t i = 0; i < NUM_POINTERS; ++i) { - umf_ipc_handle_t ipcHandle; - size_t handleSize; - ret = umfGetIPCHandle(ptrs[i], &ipcHandle, &handleSize); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - ipcHandles[i] = ipcHandle; - } - - std::array, NTHREADS> openedIpcHandles; - umf_ipc_handler_handle_t ipcHandler = nullptr; - ret = umfPoolGetIPCHandler(pool.get(), &ipcHandler); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - ASSERT_NE(ipcHandler, nullptr); - - umf_test::syncthreads_barrier syncthreads(NTHREADS); - - auto openHandlesFn = [&ipcHandles, &openedIpcHandles, &syncthreads, - ipcHandler](size_t tid) { - syncthreads(); - for (auto ipcHandle : ipcHandles) { - void *ptr; - umf_result_t ret = umfOpenIPCHandle(ipcHandler, ipcHandle, &ptr); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - openedIpcHandles[tid].push_back(ptr); - } - }; - - umf_test::parallel_exec(NTHREADS, openHandlesFn); - - auto closeHandlesFn = [&openedIpcHandles, &syncthreads](size_t tid) { - syncthreads(); - for (void *ptr : openedIpcHandles[tid]) { - umf_result_t ret = umfCloseIPCHandle(ptr); - EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - } - }; - - umf_test::parallel_exec(NTHREADS, closeHandlesFn); +TEST_P(umfIpcTest, ConcurrentOpenConcurrentCloseHandles) { + concurrentOpenConcurrentCloseHandles(false); +} - for (auto ipcHandle : ipcHandles) { - ret = umfPutIPCHandle(ipcHandle); - EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - } +TEST_P(umfIpcTest, ConcurrentOpenConcurrentCloseHandlesShuffled) { + concurrentOpenConcurrentCloseHandles(true); +} - for (void *ptr : ptrs) { - ret = umfPoolFree(pool.get(), ptr); - EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - } +TEST_P(umfIpcTest, ConcurrentOpenCloseHandles) { + concurrentOpenCloseHandles(false); +} - pool.reset(nullptr); - EXPECT_EQ(stat.getCount, stat.allocCount); - EXPECT_EQ(stat.putCount, stat.getCount); - EXPECT_EQ(stat.openCount, stat.allocCount); - EXPECT_EQ(stat.openCount, stat.closeCount); +TEST_P(umfIpcTest, ConcurrentOpenCloseHandlesShuffled) { + concurrentOpenCloseHandles(true); } TEST_P(umfIpcTest, ConcurrentDestroyIpcHandlers) { @@ -670,8 +715,8 @@ TEST_P(umfIpcTest, ConcurrentDestroyIpcHandlers) { constexpr size_t NUM_POOLS = 10; void *ptrs[NUM_ALLOCS]; void *openedPtrs[NUM_POOLS][NUM_ALLOCS]; - std::vector consumerPools; - umf::pool_unique_handle_t producerPool = makePool(); + std::vector consumerPools; + umf_test::pool_unique_handle_t producerPool = makePool(); ASSERT_NE(producerPool.get(), nullptr); for (size_t i = 0; i < NUM_POOLS; ++i) { diff --git a/test/ipc_file_prov.sh b/test/ipc_file_prov.sh index 629b2cbb79..ffb849f25a 100755 --- a/test/ipc_file_prov.sh +++ b/test/ipc_file_prov.sh @@ -9,7 +9,12 @@ set -e -FILE_NAME="/tmp/umf_file_provider_$$" +FILE_BASE="/tmp/umf_file_provider" + +# remove old SHM files (left from the previous runs, because of crashes) +rm -f ${FILE_BASE}* + +FILE_NAME="${FILE_BASE}_$$" # port should be a number from the range <1024, 65535> PORT=$(( 1024 + ( $$ % ( 65535 - 1024 )))) diff --git a/test/memoryPoolAPI.cpp b/test/memoryPoolAPI.cpp index a949b281f9..e8071a2d81 100644 --- a/test/memoryPoolAPI.cpp +++ b/test/memoryPoolAPI.cpp @@ -125,7 +125,7 @@ TEST_P(umfPoolWithCreateFlagsTest, memoryPoolWithCustomProvider) { return UMF_RESULT_SUCCESS; } }; - umf_memory_pool_ops_t pool_ops = umf::poolMakeCOps(); + umf_memory_pool_ops_t pool_ops = umf_test::poolMakeCOps(); umf_memory_pool_handle_t hPool; auto ret = umfPoolCreate(&pool_ops, hProvider, nullptr, flags, &hPool); @@ -187,8 +187,8 @@ struct tagTest : umf_test::test { createPoolChecked(umfProxyPoolOps(), provider.get(), nullptr)); } - umf::provider_unique_handle_t provider; - umf::pool_unique_handle_t pool; + umf_test::provider_unique_handle_t provider; + umf_test::pool_unique_handle_t pool; }; TEST_F(tagTest, SetAndGet) { @@ -370,7 +370,8 @@ TEST_P(poolInitializeTest, errorPropagation) { return *errorToReturn; } }; - umf_memory_pool_ops_t pool_ops = umf::poolMakeCOps(); + umf_memory_pool_ops_t pool_ops = + umf_test::poolMakeCOps(); umf_memory_pool_handle_t hPool; auto ret = umfPoolCreate(&pool_ops, hProvider, (void *)&this->GetParam(), 0, @@ -420,7 +421,7 @@ TEST_F(test, getLastFailedMemoryProvider) { const char *name; }; umf_memory_provider_ops_t provider_ops = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); auto providerUnique1 = wrapProviderUnique( createProviderChecked(&provider_ops, (void *)"provider1")); diff --git a/test/memoryProviderAPI.cpp b/test/memoryProviderAPI.cpp index 2dc7261f06..720f11b413 100644 --- a/test/memoryProviderAPI.cpp +++ b/test/memoryProviderAPI.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023-2024 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // This file contains tests for UMF provider API @@ -335,7 +335,7 @@ TEST_P(providerInitializeTest, errorPropagation) { } }; umf_memory_provider_ops_t provider_ops = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); umf_memory_provider_handle_t hProvider; auto ret = umfMemoryProviderCreate(&provider_ops, (void *)&this->GetParam(), diff --git a/test/poolFixtures.hpp b/test/poolFixtures.hpp index 6f18664f92..de5a54685a 100644 --- a/test/poolFixtures.hpp +++ b/test/poolFixtures.hpp @@ -5,11 +5,6 @@ #ifndef UMF_TEST_POOL_FIXTURES_HPP #define UMF_TEST_POOL_FIXTURES_HPP 1 -#include "pool.hpp" -#include "provider.hpp" -#include "umf/providers/provider_devdax_memory.h" -#include "utils/utils_sanitizers.h" - #include #include #include @@ -17,7 +12,14 @@ #include #include +#include +#include +#include + #include "../malloc_compliance_tests.hpp" +#include "pool.hpp" +#include "provider.hpp" +#include "utils/utils_sanitizers.h" typedef void *(*pfnPoolParamsCreate)(); typedef umf_result_t (*pfnPoolParamsDestroy)(void *); @@ -30,7 +32,7 @@ using poolCreateExtParams = pfnPoolParamsDestroy, umf_memory_provider_ops_t *, pfnProviderParamsCreate, pfnProviderParamsDestroy>; -umf::pool_unique_handle_t poolCreateExtUnique(poolCreateExtParams params) { +umf_test::pool_unique_handle_t poolCreateExtUnique(poolCreateExtParams params) { auto [pool_ops, poolParamsCreate, poolParamsDestroy, provider_ops, providerParamsCreate, providerParamsDestroy] = params; @@ -71,7 +73,7 @@ umf::pool_unique_handle_t poolCreateExtUnique(poolCreateExtParams params) { providerParamsDestroy(provider_params); } - return umf::pool_unique_handle_t(hPool, &umfPoolDestroy); + return umf_test::pool_unique_handle_t(hPool, &umfPoolDestroy); } struct umfPoolTest : umf_test::test, @@ -84,7 +86,7 @@ struct umfPoolTest : umf_test::test, void TearDown() override { test::TearDown(); } - umf::pool_unique_handle_t pool; + umf_test::pool_unique_handle_t pool; static constexpr int NTHREADS = 5; static constexpr std::array nonAlignedAllocSizes = {5, 7, 23, 55, @@ -104,7 +106,7 @@ struct umfMultiPoolTest : umf_test::test, void TearDown() override { test::TearDown(); } - std::vector pools; + std::vector pools; }; struct umfMemTest @@ -121,7 +123,7 @@ struct umfMemTest void TearDown() override { test::TearDown(); } - umf::pool_unique_handle_t pool; + umf_test::pool_unique_handle_t pool; int expectedRecycledPoolAllocs; }; @@ -493,4 +495,141 @@ TEST_P(umfPoolTest, mallocUsableSize) { } } +TEST_P(umfPoolTest, umfPoolAlignedMalloc) { +#ifdef _WIN32 + // TODO: implement support for windows + GTEST_SKIP() << "umfPoolAlignedMalloc() is not supported on Windows"; +#else /* !_WIN32 */ + umf_result_t umf_result; + void *ptr = nullptr; + const size_t size = 2 * 1024 * 1024; // 2MB + + umf_memory_pool_handle_t pool_get = pool.get(); + + if (!umf_test::isAlignedAllocSupported(pool_get)) { + GTEST_SKIP(); + } + + ptr = umfPoolAlignedMalloc(pool_get, size, utils_get_page_size()); + ASSERT_NE(ptr, nullptr); + + umf_result = umfPoolFree(pool_get, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +#endif /* !_WIN32 */ +} + +TEST_P(umfPoolTest, pool_from_ptr_whole_size_success) { +#ifdef _WIN32 + // TODO: implement support for windows + GTEST_SKIP() << "umfPoolAlignedMalloc() is not supported on Windows"; +#else /* !_WIN32 */ + umf_result_t umf_result; + size_t size_of_pool_from_ptr; + void *ptr_for_pool = nullptr; + void *ptr = nullptr; + + umf_memory_pool_handle_t pool_get = pool.get(); + const size_t size_of_first_alloc = 2 * 1024 * 1024; // 2MB + + if (!umf_test::isAlignedAllocSupported(pool_get)) { + GTEST_SKIP(); + } + + ptr_for_pool = umfPoolAlignedMalloc(pool_get, size_of_first_alloc, + utils_get_page_size()); + ASSERT_NE(ptr_for_pool, nullptr); + + // Create provider parameters + size_of_pool_from_ptr = size_of_first_alloc; // whole size + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result = umfFixedMemoryProviderParamsCreate(¶ms, ptr_for_pool, + size_of_pool_from_ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + umf_memory_provider_handle_t providerFromPtr = nullptr; + umf_result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + &providerFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(providerFromPtr, nullptr); + + umf_memory_pool_handle_t poolFromPtr = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), providerFromPtr, nullptr, 0, + &poolFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ptr = umfPoolMalloc(poolFromPtr, size_of_pool_from_ptr); + ASSERT_NE(ptr, nullptr); + + memset(ptr, 0xFF, size_of_pool_from_ptr); + + umf_result = umfPoolFree(poolFromPtr, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(poolFromPtr); + umfMemoryProviderDestroy(providerFromPtr); + umfFixedMemoryProviderParamsDestroy(params); + + umf_result = umfPoolFree(pool_get, ptr_for_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +#endif /* !_WIN32 */ +} + +TEST_P(umfPoolTest, pool_from_ptr_half_size_success) { +#ifdef _WIN32 + // TODO: implement support for windows + GTEST_SKIP() << "umfPoolAlignedMalloc() is not supported on Windows"; +#else /* !_WIN32 */ + umf_result_t umf_result; + size_t size_of_pool_from_ptr; + void *ptr_for_pool = nullptr; + void *ptr = nullptr; + + umf_memory_pool_handle_t pool_get = pool.get(); + const size_t size_of_first_alloc = 2 * 1024 * 1024; // 2MB + + if (!umf_test::isAlignedAllocSupported(pool_get)) { + GTEST_SKIP(); + } + + ptr_for_pool = umfPoolAlignedMalloc(pool_get, size_of_first_alloc, + utils_get_page_size()); + ASSERT_NE(ptr_for_pool, nullptr); + + // Create provider parameters + size_of_pool_from_ptr = size_of_first_alloc / 2; // half size + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result = umfFixedMemoryProviderParamsCreate(¶ms, ptr_for_pool, + size_of_pool_from_ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + umf_memory_provider_handle_t providerFromPtr = nullptr; + umf_result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + &providerFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(providerFromPtr, nullptr); + + umf_memory_pool_handle_t poolFromPtr = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), providerFromPtr, nullptr, 0, + &poolFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ptr = umfPoolMalloc(poolFromPtr, size_of_pool_from_ptr); + ASSERT_NE(ptr, nullptr); + + memset(ptr, 0xFF, size_of_pool_from_ptr); + + umf_result = umfPoolFree(poolFromPtr, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(poolFromPtr); + umfMemoryProviderDestroy(providerFromPtr); + umfFixedMemoryProviderParamsDestroy(params); + + umf_result = umfPoolFree(pool_get, ptr_for_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +#endif /* !_WIN32 */ +} + #endif /* UMF_TEST_POOL_FIXTURES_HPP */ diff --git a/test/pools/disjoint_pool.cpp b/test/pools/disjoint_pool.cpp index 02f7698024..9bdef4f131 100644 --- a/test/pools/disjoint_pool.cpp +++ b/test/pools/disjoint_pool.cpp @@ -40,7 +40,7 @@ TEST_F(test, internals) { } }; umf_memory_provider_ops_t provider_ops = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); auto providerUnique = wrapProviderUnique(createProviderChecked(&provider_ops, nullptr)); @@ -113,9 +113,8 @@ TEST_F(test, internals) { EXPECT_GE(slab->num_chunks_total, slab->slab_size / bucket->size); // check allocation in slab - EXPECT_EQ(slab->chunks[0], true); - EXPECT_EQ(slab->chunks[1], false); - EXPECT_EQ(slab->first_free_chunk_idx, 1); + EXPECT_EQ(slab_read_chunk_bit(slab, 0), false); + EXPECT_EQ(slab_read_chunk_bit(slab, 1), true); // TODO: // * multiple alloc + free from single bucket @@ -152,7 +151,7 @@ TEST_F(test, freeErrorPropagation) { } }; umf_memory_provider_ops_t provider_ops = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); auto providerUnique = wrapProviderUnique(createProviderChecked(&provider_ops, nullptr)); @@ -207,7 +206,7 @@ TEST_F(test, sharedLimits) { } }; umf_memory_provider_ops_t provider_ops = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); static constexpr size_t SlabMinSize = 1024; static constexpr size_t MaxSize = 4 * SlabMinSize; diff --git a/test/pools/jemalloc_pool.cpp b/test/pools/jemalloc_pool.cpp index e282be3161..8112f36bf4 100644 --- a/test/pools/jemalloc_pool.cpp +++ b/test/pools/jemalloc_pool.cpp @@ -11,9 +11,7 @@ using umf_test::test; using namespace umf_test; -using os_params_unique_handle_t = - std::unique_ptr; +using void_unique_ptr = std::unique_ptr; void *createOsMemoryProviderParams() { umf_os_memory_provider_params_handle_t params = nullptr; @@ -30,11 +28,43 @@ umf_result_t destroyOsMemoryProviderParams(void *params) { (umf_os_memory_provider_params_handle_t)params); } +void *createFixedMemoryProviderParams() { + // Allocate a memory buffer to use with the fixed memory provider. + // The umfPoolTest.malloc_compliance test requires a lot of memory. + size_t memory_size = (1UL << 31); + static void_unique_ptr memory_buffer = + void_unique_ptr(malloc(memory_size), free); + if (memory_buffer.get() == NULL) { + throw std::runtime_error( + "Failed to allocate memory for Fixed memory provider"); + } + + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result_t res = umfFixedMemoryProviderParamsCreate( + ¶ms, memory_buffer.get(), memory_size); + if (res != UMF_RESULT_SUCCESS) { + throw std::runtime_error( + "Failed to create Fixed memory provider params"); + } + + return params; +} + +umf_result_t destroyFixedMemoryProviderParams(void *params) { + return umfFixedMemoryProviderParamsDestroy( + (umf_fixed_memory_provider_params_handle_t)params); +} + INSTANTIATE_TEST_SUITE_P( jemallocPoolTest, umfPoolTest, - ::testing::Values(poolCreateExtParams{ - umfJemallocPoolOps(), nullptr, nullptr, umfOsMemoryProviderOps(), - createOsMemoryProviderParams, destroyOsMemoryProviderParams})); + ::testing::Values(poolCreateExtParams{umfJemallocPoolOps(), nullptr, + nullptr, umfOsMemoryProviderOps(), + createOsMemoryProviderParams, + destroyOsMemoryProviderParams}, + poolCreateExtParams{umfJemallocPoolOps(), nullptr, + nullptr, umfFixedMemoryProviderOps(), + createFixedMemoryProviderParams, + destroyFixedMemoryProviderParams})); // this test makes sure that jemalloc does not use // memory provider to allocate metadata (and hence diff --git a/test/pools/pool_base_alloc.cpp b/test/pools/pool_base_alloc.cpp index ca931bcec3..441ab37ec3 100644 --- a/test/pools/pool_base_alloc.cpp +++ b/test/pools/pool_base_alloc.cpp @@ -17,17 +17,17 @@ struct base_alloc_pool : public umf_test::pool_base_t { void *malloc(size_t size) noexcept { return umf_ba_global_alloc(size); } void *calloc(size_t, size_t) noexcept { - umf::getPoolLastStatusRef() = + umf_test::getPoolLastStatusRef() = UMF_RESULT_ERROR_NOT_SUPPORTED; return NULL; } void *realloc(void *, size_t) noexcept { - umf::getPoolLastStatusRef() = + umf_test::getPoolLastStatusRef() = UMF_RESULT_ERROR_NOT_SUPPORTED; return NULL; } void *aligned_malloc(size_t, size_t) noexcept { - umf::getPoolLastStatusRef() = + umf_test::getPoolLastStatusRef() = UMF_RESULT_ERROR_NOT_SUPPORTED; return NULL; } @@ -39,11 +39,12 @@ struct base_alloc_pool : public umf_test::pool_base_t { return UMF_RESULT_SUCCESS; } umf_result_t get_last_allocation_error() { - return umf::getPoolLastStatusRef(); + return umf_test::getPoolLastStatusRef(); } }; -umf_memory_pool_ops_t BA_POOL_OPS = umf::poolMakeCOps(); +umf_memory_pool_ops_t BA_POOL_OPS = + umf_test::poolMakeCOps(); INSTANTIATE_TEST_SUITE_P(baPool, umfPoolTest, ::testing::Values(poolCreateExtParams{ diff --git a/test/pools/scalable_pool.cpp b/test/pools/scalable_pool.cpp index 14cf5f3059..54c0128a40 100644 --- a/test/pools/scalable_pool.cpp +++ b/test/pools/scalable_pool.cpp @@ -61,7 +61,7 @@ struct umfScalablePoolParamsTest }; static constexpr umf_memory_provider_ops_t VALIDATOR_PROVIDER_OPS = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); umfScalablePoolParamsTest() : expected_params{0, false}, params(nullptr) {} void SetUp() override { @@ -82,7 +82,7 @@ struct umfScalablePoolParamsTest test::TearDown(); } - umf::pool_unique_handle_t makePool() { + umf_test::pool_unique_handle_t makePool() { umf_memory_provider_handle_t hProvider = nullptr; umf_memory_pool_handle_t hPool = nullptr; @@ -94,7 +94,7 @@ struct umfScalablePoolParamsTest UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &hPool); EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - return umf::pool_unique_handle_t(hPool, &umfPoolDestroy); + return umf_test::pool_unique_handle_t(hPool, &umfPoolDestroy); } void allocFreeFlow() { diff --git a/test/provider_devdax_memory.cpp b/test/provider_devdax_memory.cpp index 7765dd08da..6efeef90cb 100644 --- a/test/provider_devdax_memory.cpp +++ b/test/provider_devdax_memory.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2024 Intel Corporation +// Copyright (C) 2024-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -11,8 +11,8 @@ #include "base.hpp" -#include "cpp_helpers.hpp" #include "test_helpers.h" +#include "utils/cpp_helpers.hpp" #include #include @@ -46,7 +46,7 @@ static int compare_native_error_str(const char *message, int error) { using providerCreateExtParams = std::tuple; static void providerCreateExt(providerCreateExtParams params, - umf::provider_unique_handle_t *handle) { + umf_test::provider_unique_handle_t *handle) { umf_memory_provider_handle_t hProvider = nullptr; auto [provider_ops, provider_params] = params; @@ -55,8 +55,8 @@ static void providerCreateExt(providerCreateExtParams params, ASSERT_EQ(ret, UMF_RESULT_SUCCESS); ASSERT_NE(hProvider, nullptr); - *handle = - umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); + *handle = umf_test::provider_unique_handle_t(hProvider, + &umfMemoryProviderDestroy); } struct umfProviderTest @@ -74,7 +74,7 @@ struct umfProviderTest void TearDown() override { test::TearDown(); } - umf::provider_unique_handle_t provider; + umf_test::provider_unique_handle_t provider; size_t page_size; size_t page_plus_64; }; diff --git a/test/provider_file_memory.cpp b/test/provider_file_memory.cpp index cfa37be31a..bcc9d26453 100644 --- a/test/provider_file_memory.cpp +++ b/test/provider_file_memory.cpp @@ -1,11 +1,11 @@ -// Copyright (C) 2024 Intel Corporation +// Copyright (C) 2024-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "base.hpp" -#include "cpp_helpers.hpp" #include "test_helpers.h" +#include "utils/cpp_helpers.hpp" #ifndef _WIN32 #include "test_helpers_linux.h" #endif @@ -42,7 +42,7 @@ static int compare_native_error_str(const char *message, int error) { using providerCreateExtParams = std::tuple; static void providerCreateExt(providerCreateExtParams params, - umf::provider_unique_handle_t *handle) { + umf_test::provider_unique_handle_t *handle) { umf_memory_provider_handle_t hProvider = nullptr; auto [provider_ops, provider_params] = params; @@ -51,8 +51,8 @@ static void providerCreateExt(providerCreateExtParams params, ASSERT_EQ(ret, UMF_RESULT_SUCCESS); ASSERT_NE(hProvider, nullptr); - *handle = - umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); + *handle = umf_test::provider_unique_handle_t(hProvider, + &umfMemoryProviderDestroy); } struct FileProviderParamsDefault @@ -70,7 +70,7 @@ struct FileProviderParamsDefault void TearDown() override { test::TearDown(); } - umf::provider_unique_handle_t provider; + umf_test::provider_unique_handle_t provider; size_t page_size; size_t page_plus_64; }; diff --git a/test/provider_fixed_memory.cpp b/test/provider_fixed_memory.cpp index 7f976a1f5d..dac651435a 100644 --- a/test/provider_fixed_memory.cpp +++ b/test/provider_fixed_memory.cpp @@ -1,20 +1,22 @@ -// Copyright (C) 2024 Intel Corporation +// Copyright (C) 2024-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "base.hpp" -#include "cpp_helpers.hpp" #include "test_helpers.h" +#include "utils/cpp_helpers.hpp" #ifndef _WIN32 #include "test_helpers_linux.h" #endif #include +#include #include using umf_test::test; +#define FIXED_BUFFER_SIZE (10 * utils_get_page_size()) #define INVALID_PTR ((void *)0x01) typedef enum purge_t { @@ -39,7 +41,7 @@ static int compare_native_error_str(const char *message, int error) { using providerCreateExtParams = std::tuple; static void providerCreateExt(providerCreateExtParams params, - umf::provider_unique_handle_t *handle) { + umf_test::provider_unique_handle_t *handle) { umf_memory_provider_handle_t hProvider = nullptr; auto [provider_ops, provider_params] = params; @@ -48,8 +50,8 @@ static void providerCreateExt(providerCreateExtParams params, ASSERT_EQ(ret, UMF_RESULT_SUCCESS); ASSERT_NE(hProvider, nullptr); - *handle = - umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); + *handle = umf_test::provider_unique_handle_t(hProvider, + &umfMemoryProviderDestroy); } struct FixedProviderTest @@ -59,7 +61,7 @@ struct FixedProviderTest test::SetUp(); // Allocate a memory buffer to use with the fixed memory provider - memory_size = utils_get_page_size() * 10; // Allocate 10 pages + memory_size = FIXED_BUFFER_SIZE; // Allocate 10 pages memory_buffer = malloc(memory_size); ASSERT_NE(memory_buffer, nullptr); @@ -136,7 +138,7 @@ struct FixedProviderTest } } - umf::provider_unique_handle_t provider; + umf_test::provider_unique_handle_t provider; size_t page_size; size_t page_plus_64; void *memory_buffer = nullptr; @@ -391,3 +393,109 @@ TEST_P(FixedProviderTest, split) { umf_result = umfMemoryProviderFree(provider.get(), ptr2, size); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } + +TEST_P(FixedProviderTest, pool_from_ptr_whole_size_success) { + umf_result_t umf_result; + size_t size_of_first_alloc; + size_t size_of_pool_from_ptr; + void *ptr_for_pool = nullptr; + void *ptr = nullptr; + + umf_memory_pool_handle_t proxyFixedPool = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), provider.get(), nullptr, 0, + &proxyFixedPool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + size_of_first_alloc = FIXED_BUFFER_SIZE - (2 * page_size); + ptr_for_pool = umfPoolMalloc(proxyFixedPool, size_of_first_alloc); + ASSERT_NE(ptr_for_pool, nullptr); + + // Create provider parameters + size_of_pool_from_ptr = size_of_first_alloc; // whole size + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result = umfFixedMemoryProviderParamsCreate(¶ms, ptr_for_pool, + size_of_pool_from_ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + umf_memory_provider_handle_t providerFromPtr = nullptr; + umf_result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + &providerFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(providerFromPtr, nullptr); + + umf_memory_pool_handle_t poolFromPtr = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), providerFromPtr, nullptr, 0, + &poolFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ptr = umfPoolMalloc(poolFromPtr, size_of_pool_from_ptr); + ASSERT_NE(ptr, nullptr); + + memset(ptr, 0xFF, size_of_pool_from_ptr); + + umf_result = umfPoolFree(poolFromPtr, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(poolFromPtr); + umfMemoryProviderDestroy(providerFromPtr); + umfFixedMemoryProviderParamsDestroy(params); + + umf_result = umfPoolFree(proxyFixedPool, ptr_for_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(proxyFixedPool); +} + +TEST_P(FixedProviderTest, pool_from_ptr_half_size_success) { + umf_result_t umf_result; + size_t size_of_first_alloc; + size_t size_of_pool_from_ptr; + void *ptr_for_pool = nullptr; + void *ptr = nullptr; + + umf_memory_pool_handle_t proxyFixedPool = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), provider.get(), nullptr, 0, + &proxyFixedPool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + size_of_first_alloc = FIXED_BUFFER_SIZE - (2 * page_size); + ptr_for_pool = umfPoolMalloc(proxyFixedPool, size_of_first_alloc); + ASSERT_NE(ptr_for_pool, nullptr); + + // Create provider parameters + size_of_pool_from_ptr = size_of_first_alloc / 2; // half size + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result = umfFixedMemoryProviderParamsCreate(¶ms, ptr_for_pool, + size_of_pool_from_ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + umf_memory_provider_handle_t providerFromPtr = nullptr; + umf_result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + &providerFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(providerFromPtr, nullptr); + + umf_memory_pool_handle_t poolFromPtr = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), providerFromPtr, nullptr, 0, + &poolFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ptr = umfPoolMalloc(poolFromPtr, size_of_pool_from_ptr); + ASSERT_NE(ptr, nullptr); + + memset(ptr, 0xFF, size_of_pool_from_ptr); + + umf_result = umfPoolFree(poolFromPtr, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(poolFromPtr); + umfMemoryProviderDestroy(providerFromPtr); + umfFixedMemoryProviderParamsDestroy(params); + + umf_result = umfPoolFree(proxyFixedPool, ptr_for_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(proxyFixedPool); +} diff --git a/test/provider_os_memory.cpp b/test/provider_os_memory.cpp index 5b647b642c..f3552b9236 100644 --- a/test/provider_os_memory.cpp +++ b/test/provider_os_memory.cpp @@ -4,9 +4,9 @@ #include "base.hpp" -#include "cpp_helpers.hpp" #include "ipcFixtures.hpp" #include "test_helpers.h" +#include "utils/cpp_helpers.hpp" #include #include @@ -47,7 +47,7 @@ static int compare_native_error_str(const char *message, int error) { using providerCreateExtParams = std::tuple; static void providerCreateExt(providerCreateExtParams params, - umf::provider_unique_handle_t *handle) { + umf_test::provider_unique_handle_t *handle) { umf_memory_provider_handle_t hProvider = nullptr; auto [provider_ops, provider_params] = params; @@ -56,8 +56,8 @@ static void providerCreateExt(providerCreateExtParams params, ASSERT_EQ(ret, UMF_RESULT_SUCCESS); ASSERT_NE(hProvider, nullptr); - *handle = - umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); + *handle = umf_test::provider_unique_handle_t(hProvider, + &umfMemoryProviderDestroy); } struct umfProviderTest @@ -75,7 +75,7 @@ struct umfProviderTest void TearDown() override { test::TearDown(); } - umf::provider_unique_handle_t provider; + umf_test::provider_unique_handle_t provider; size_t page_size; size_t page_plus_64; }; diff --git a/test/provider_tracking.cpp b/test/provider_tracking.cpp new file mode 100644 index 0000000000..55acc452cf --- /dev/null +++ b/test/provider_tracking.cpp @@ -0,0 +1,374 @@ +// Copyright (C) 2025 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "base.hpp" + +#include "test_helpers.h" +#include "utils/cpp_helpers.hpp" +#ifndef _WIN32 +#include "test_helpers_linux.h" +#endif + +#include +#include +#include + +using umf_test::test; + +#define FIXED_BUFFER_SIZE (512 * utils_get_page_size()) +#define INVALID_PTR ((void *)0x01) + +using providerCreateExtParams = std::tuple; + +static void providerCreateExt(providerCreateExtParams params, + umf_test::provider_unique_handle_t *handle) { + umf_memory_provider_handle_t hProvider = nullptr; + auto [provider_ops, provider_params] = params; + + auto ret = + umfMemoryProviderCreate(provider_ops, provider_params, &hProvider); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(hProvider, nullptr); + + *handle = umf_test::provider_unique_handle_t(hProvider, + &umfMemoryProviderDestroy); +} + +struct TrackingProviderTest + : umf_test::test, + ::testing::WithParamInterface { + void SetUp() override { + test::SetUp(); + + // Allocate a memory buffer to use with the fixed memory provider + memory_size = FIXED_BUFFER_SIZE; + memory_buffer = malloc(memory_size); + ASSERT_NE(memory_buffer, nullptr); + + // Create provider parameters + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result_t res = umfFixedMemoryProviderParamsCreate( + ¶ms, memory_buffer, memory_size); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + providerCreateExt(std::make_tuple(umfFixedMemoryProviderOps(), params), + &provider); + + umfFixedMemoryProviderParamsDestroy(params); + umf_result_t umf_result = + umfMemoryProviderGetMinPageSize(provider.get(), NULL, &page_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + page_plus_64 = page_size + 64; + + umf_memory_pool_handle_t hPool = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), provider.get(), nullptr, + 0, &hPool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + pool = umf_test::pool_unique_handle_t(hPool, &umfPoolDestroy); + } + + void TearDown() override { + if (memory_buffer) { + free(memory_buffer); + memory_buffer = nullptr; + } + test::TearDown(); + } + + umf_test::provider_unique_handle_t provider; + umf_test::pool_unique_handle_t pool; + size_t page_size; + size_t page_plus_64; + void *memory_buffer = nullptr; + size_t memory_size = 0; +}; + +static void +createPoolFromAllocation(void *ptr0, size_t size1, + umf_memory_provider_handle_t *_providerFromPtr, + umf_memory_pool_handle_t *_poolFromPtr) { + umf_result_t umf_result; + + // Create provider parameters + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result = umfFixedMemoryProviderParamsCreate(¶ms, ptr0, size1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + umf_memory_provider_handle_t provider1 = nullptr; + umf_result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + &provider1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider1, nullptr); + + umf_memory_pool_handle_t pool1 = nullptr; + umf_result = + umfPoolCreate(umfProxyPoolOps(), provider1, nullptr, 0, &pool1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfFixedMemoryProviderParamsDestroy(params); + + *_providerFromPtr = provider1; + *_poolFromPtr = pool1; +} + +// TESTS + +INSTANTIATE_TEST_SUITE_P(trackingProviderTest, TrackingProviderTest, + ::testing::Values(providerCreateExtParams{ + umfFixedMemoryProviderOps(), nullptr})); + +TEST_P(TrackingProviderTest, create_destroy) { + // Creation and destruction are handled in SetUp and TearDown +} + +TEST_P(TrackingProviderTest, whole_size_success) { + umf_result_t umf_result; + size_t size0; + size_t size1; + void *ptr0 = nullptr; + void *ptr1 = nullptr; + + umf_memory_pool_handle_t pool0 = pool.get(); + + size0 = FIXED_BUFFER_SIZE - (2 * page_size); + ptr0 = umfPoolAlignedMalloc(pool0, size0, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + size1 = size0; // whole size + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size1, &provider1, &pool1); + + ptr1 = umfPoolMalloc(pool1, size1); + ASSERT_NE(ptr1, nullptr); + + umf_result = umfPoolFree(pool1, ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(pool1); + umfMemoryProviderDestroy(provider1); + + umf_result = umfPoolFree(pool0, ptr0); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(TrackingProviderTest, half_size_success) { + umf_result_t umf_result; + size_t size0; + size_t size1; + void *ptr0 = nullptr; + void *ptr1 = nullptr; + + umf_memory_pool_handle_t pool0 = pool.get(); + + size0 = FIXED_BUFFER_SIZE - (2 * page_size); + ptr0 = umfPoolAlignedMalloc(pool0, size0, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + size1 = size0 / 2; // half size + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size1, &provider1, &pool1); + + ptr1 = umfPoolMalloc(pool1, size1); + ASSERT_NE(ptr1, nullptr); + + umf_result = umfPoolFree(pool1, ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(pool1); + umfMemoryProviderDestroy(provider1); + + umf_result = umfPoolFree(pool0, ptr0); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(TrackingProviderTest, failure_exceeding_size) { + umf_result_t umf_result; + size_t size0; + size_t size1; + void *ptr0 = nullptr; + void *ptr1 = nullptr; + + umf_memory_pool_handle_t pool0 = pool.get(); + + size0 = FIXED_BUFFER_SIZE - (2 * page_size); + ptr0 = umfPoolAlignedMalloc(pool0, size0, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + size1 = FIXED_BUFFER_SIZE - page_size; // exceeding size + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size1, &provider1, &pool1); + + ptr1 = umfPoolMalloc(pool1, size1); + ASSERT_EQ(ptr1, nullptr); + + umfPoolDestroy(pool1); + umfMemoryProviderDestroy(provider1); + + umf_result = umfPoolFree(pool0, ptr0); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +#define MAX_ARRAY 9 +#define TEST_LEVEL_SUCCESS 7 +#define TEST_LEVEL_FAILURE 8 + +TEST_P(TrackingProviderTest, success_max_levels) { + umf_result_t umf_result; + size_t size; + void *ptr[MAX_ARRAY] = {0}; + umf_memory_provider_handle_t providers[MAX_ARRAY] = {0}; + umf_memory_pool_handle_t pools[MAX_ARRAY] = {0}; + + size = FIXED_BUFFER_SIZE - (2 * page_size); + pools[0] = pool.get(); + + for (int i = 0; i < TEST_LEVEL_SUCCESS; i++) { + fprintf(stderr, "Alloc #%d\n", i); + ptr[i] = umfPoolAlignedMalloc(pools[i], size, utils_get_page_size()); + ASSERT_NE(ptr[i], nullptr); + + createPoolFromAllocation(ptr[i], size, &providers[i + 1], + &pools[i + 1]); + } + + int s = TEST_LEVEL_SUCCESS; + fprintf(stderr, "Alloc #%d\n", s); + ptr[s] = umfPoolAlignedMalloc(pools[s], size, utils_get_page_size()); + ASSERT_NE(ptr[s], nullptr); + + fprintf(stderr, "Free #%d\n", s); + umf_result = umfPoolFree(pools[s], ptr[s]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + for (int i = TEST_LEVEL_SUCCESS - 1; i >= 0; i--) { + umfPoolDestroy(pools[i + 1]); + umfMemoryProviderDestroy(providers[i + 1]); + + fprintf(stderr, "Free #%d\n", i); + umf_result = umfPoolFree(pools[i], ptr[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } +} + +TEST_P(TrackingProviderTest, failure_exceeding_levels) { + umf_result_t umf_result; + size_t size; + void *ptr[MAX_ARRAY] = {0}; + umf_memory_provider_handle_t providers[MAX_ARRAY] = {0}; + umf_memory_pool_handle_t pools[MAX_ARRAY] = {0}; + + size = FIXED_BUFFER_SIZE - (2 * page_size); + pools[0] = pool.get(); + + for (int i = 0; i < TEST_LEVEL_FAILURE; i++) { + fprintf(stderr, "Alloc #%d\n", i); + ptr[i] = umfPoolAlignedMalloc(pools[i], size, utils_get_page_size()); + ASSERT_NE(ptr[i], nullptr); + + createPoolFromAllocation(ptr[i], size, &providers[i + 1], + &pools[i + 1]); + } + + // tracker level is too high + int f = TEST_LEVEL_FAILURE; + fprintf(stderr, "Alloc #%d\n", f); + ptr[f] = umfPoolAlignedMalloc(pools[f], size, utils_get_page_size()); + ASSERT_EQ(ptr[f], nullptr); + + for (int i = TEST_LEVEL_FAILURE - 1; i >= 0; i--) { + umfPoolDestroy(pools[i + 1]); + umfMemoryProviderDestroy(providers[i + 1]); + + fprintf(stderr, "Free #%d\n", i); + umf_result = umfPoolFree(pools[i], ptr[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } +} + +TEST_P(TrackingProviderTest, reverted_free_half_size) { + umf_result_t umf_result; + size_t size0; + size_t size1; + void *ptr0 = nullptr; + void *ptr1 = nullptr; + + umf_memory_pool_handle_t pool0 = pool.get(); + + size0 = FIXED_BUFFER_SIZE - (2 * page_size); + ptr0 = umfPoolAlignedMalloc(pool0, size0, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size0, &provider1, &pool1); + + size1 = size0 / 2; // half size + + ptr1 = umfPoolMalloc(pool1, size1); + ASSERT_NE(ptr1, nullptr); + + // Freeing the "busy" pointer from the first pool is an Undefined Behavior + // It fails now if the sizes are different. + // see: https://github.com/oneapi-src/unified-memory-framework/pull/1161 + umf_result = umfPoolFree(pool0, ptr0); + + umf_result = umfPoolFree(pool1, ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(pool1); + umfMemoryProviderDestroy(provider1); + + // It could have been freed above, + // so we cannot verify the result here. + umf_result = umfPoolFree(pool0, ptr0); +} + +TEST_P(TrackingProviderTest, reverted_free_the_same_size) { + umf_result_t umf_result; + size_t size0; + size_t size1; + void *ptr0 = nullptr; + void *ptr1 = nullptr; + + umf_memory_pool_handle_t pool0 = pool.get(); + + size0 = FIXED_BUFFER_SIZE - (2 * page_size); + ptr0 = umfPoolAlignedMalloc(pool0, size0, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size0, &provider1, &pool1); + + size1 = size0; // the same size + + ptr1 = umfPoolMalloc(pool1, size1); + ASSERT_NE(ptr1, nullptr); + + // Freeing the "busy" pointer from the first pool is an Undefined Behavior + // It succeeds now if the sizes are equal. + // see: https://github.com/oneapi-src/unified-memory-framework/pull/1161 + umf_result = umfPoolFree(pool0, ptr0); + + // try to free the pointer from the second pool (the same size) + umf_result = umfPoolFree(pool1, ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(pool1); + umfMemoryProviderDestroy(provider1); + + // It could have been freed above, + // so we cannot verify the result here. + umf_result = umfPoolFree(pool0, ptr0); +} diff --git a/test/provider_tracking_fixture_tests.cpp b/test/provider_tracking_fixture_tests.cpp new file mode 100644 index 0000000000..d81d4f8b1d --- /dev/null +++ b/test/provider_tracking_fixture_tests.cpp @@ -0,0 +1,91 @@ +// Copyright (C) 2025 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include + +#include "base.hpp" +#include "provider.hpp" + +#include "test_helpers.h" +#include "utils/cpp_helpers.hpp" +#ifndef _WIN32 +#include "test_helpers_linux.h" +#endif + +#include "poolFixtures.hpp" + +#define FILE_PATH ((char *)"tmp_file") + +struct provider_from_pool : public umf_test::provider_base_t { + umf_memory_pool_handle_t pool; + umf_result_t initialize(umf_memory_pool_handle_t _pool) noexcept { + if (!_pool) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + pool = _pool; + return UMF_RESULT_SUCCESS; + } + umf_result_t alloc(size_t size, size_t align, void **ptr) noexcept { + *ptr = umfPoolAlignedMalloc(pool, size, align); + return (*ptr) ? UMF_RESULT_SUCCESS + : UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + umf_result_t free(void *ptr, size_t) noexcept { + return umfPoolFree(pool, ptr); + } + const char *get_name() noexcept { return "provider_from_pool"; } + + virtual ~provider_from_pool() { + if (pool) { + umfPoolDestroy(pool); + pool = nullptr; + } + } +}; + +umf_memory_provider_ops_t PROVIDER_FROM_POOL_OPS = + umf_test::providerMakeCOps(); + +static void *providerFromPoolParamsCreate(void) { + umf_file_memory_provider_params_handle_t paramsFile = NULL; + umf_result_t umf_result = + umfFileMemoryProviderParamsCreate(¶msFile, FILE_PATH); + EXPECT_EQ(umf_result, UMF_RESULT_SUCCESS); + EXPECT_NE(paramsFile, nullptr); + + umf_memory_provider_handle_t providerFile = nullptr; + umf_result = umfMemoryProviderCreate(umfFileMemoryProviderOps(), paramsFile, + &providerFile); + EXPECT_EQ(umf_result, UMF_RESULT_SUCCESS); + EXPECT_NE(providerFile, nullptr); + + umf_memory_pool_handle_t poolProxyFile = nullptr; + umf_result = + umfPoolCreate(umfProxyPoolOps(), providerFile, nullptr, + UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &poolProxyFile); + EXPECT_EQ(umf_result, UMF_RESULT_SUCCESS); + EXPECT_NE(poolProxyFile, nullptr); + + umf_result = umfFileMemoryProviderParamsDestroy(paramsFile); + EXPECT_EQ(umf_result, UMF_RESULT_SUCCESS); + paramsFile = nullptr; + + return poolProxyFile; +} + +// TESTS + +INSTANTIATE_TEST_SUITE_P(TrackingProviderPoolTest, umfPoolTest, + ::testing::Values(poolCreateExtParams{ + umfProxyPoolOps(), nullptr, nullptr, + &PROVIDER_FROM_POOL_OPS, + providerFromPoolParamsCreate, nullptr})); + +INSTANTIATE_TEST_SUITE_P(TrackingProviderMultiPoolTest, umfMultiPoolTest, + ::testing::Values(poolCreateExtParams{ + umfProxyPoolOps(), nullptr, nullptr, + &PROVIDER_FROM_POOL_OPS, + providerFromPoolParamsCreate, nullptr})); diff --git a/test/providers/cuda_helpers.cpp b/test/providers/cuda_helpers.cpp index a607d7ecb3..3e81c184ff 100644 --- a/test/providers/cuda_helpers.cpp +++ b/test/providers/cuda_helpers.cpp @@ -412,6 +412,18 @@ CUcontext get_mem_context(void *ptr) { return context; } +int get_mem_device(void *ptr) { + int device; + CUresult res = libcu_ops.cuPointerGetAttribute( + &device, CU_POINTER_ATTRIBUTE_DEVICE_ORDINAL, (CUdeviceptr)ptr); + if (res != CUDA_SUCCESS) { + fprintf(stderr, "cuPointerGetAttribute() failed!\n"); + return -1; + } + + return device; +} + CUcontext get_current_context() { CUcontext context; CUresult res = libcu_ops.cuCtxGetCurrent(&context); diff --git a/test/providers/cuda_helpers.h b/test/providers/cuda_helpers.h index e7deb9064c..944e6dbef8 100644 --- a/test/providers/cuda_helpers.h +++ b/test/providers/cuda_helpers.h @@ -48,6 +48,8 @@ unsigned int get_mem_host_alloc_flags(void *ptr); CUcontext get_mem_context(void *ptr); +int get_mem_device(void *ptr); + CUcontext get_current_context(); #ifdef __cplusplus diff --git a/test/providers/provider_cuda.cpp b/test/providers/provider_cuda.cpp index 9c7f76dd10..a7e5dbe5a0 100644 --- a/test/providers/provider_cuda.cpp +++ b/test/providers/provider_cuda.cpp @@ -142,14 +142,15 @@ struct umfCUDAProviderTest memAccessor = nullptr; expected_context = cudaTestHelper.get_test_context(); + expected_device = cudaTestHelper.get_test_device(); params = create_cuda_prov_params(cudaTestHelper.get_test_context(), cudaTestHelper.get_test_device(), memory_type, 0 /* alloc flags */); ASSERT_NE(expected_context, nullptr); + ASSERT_GE(expected_device, 0); switch (memory_type) { case UMF_MEMORY_TYPE_DEVICE: - memAccessor = std::make_unique( cudaTestHelper.get_test_context(), cudaTestHelper.get_test_device()); @@ -178,6 +179,7 @@ struct umfCUDAProviderTest std::unique_ptr memAccessor = nullptr; CUcontext expected_context = nullptr; + int expected_device = -1; umf_usm_memory_type_t expected_memory_type; }; @@ -328,6 +330,44 @@ TEST_P(umfCUDAProviderTest, getPageSizeInvalidArgs) { umfMemoryProviderDestroy(provider); } +TEST_P(umfCUDAProviderTest, cudaProviderDefaultParams) { + umf_cuda_memory_provider_params_handle_t defaultParams = nullptr; + umf_result_t umf_result = umfCUDAMemoryProviderParamsCreate(&defaultParams); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_result = umfCUDAMemoryProviderParamsSetMemoryType(defaultParams, + expected_memory_type); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + // NOTE: we intentionally do not set any context and device params + + umf_memory_provider_handle_t provider = nullptr; + umf_result = umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), + defaultParams, &provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider, nullptr); + + // do single alloc and check if the context and device id of allocated + // memory are correct + + void *ptr = nullptr; + umf_result = umfMemoryProviderAlloc(provider, 128, 0, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + + CUcontext actual_mem_context = get_mem_context(ptr); + ASSERT_EQ(actual_mem_context, expected_context); + + int actual_device = get_mem_device(ptr); + ASSERT_EQ(actual_device, expected_device); + + umf_result = umfMemoryProviderFree(provider, ptr, 128); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfMemoryProviderDestroy(provider); + umfCUDAMemoryProviderParamsDestroy(defaultParams); +} + TEST_P(umfCUDAProviderTest, cudaProviderNullParams) { umf_result_t res = umfCUDAMemoryProviderParamsCreate(nullptr); EXPECT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); diff --git a/test/supp/drd-test_ipc_max_opened_limit.supp b/test/supp/drd-test_ipc_max_opened_limit.supp new file mode 100644 index 0000000000..fbdbd0183f --- /dev/null +++ b/test/supp/drd-test_ipc_max_opened_limit.supp @@ -0,0 +1,34 @@ +{ + Conditional variable destruction false-positive + drd:CondErr + ... + fun:pthread_cond_destroy@* + ... +} + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + drd:ConflictingAccess + fun:utils_atomic_load_acquire_ptr + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} + +{ + [false-positive] trackingGetIpcHandle + drd:ConflictingAccess + fun:trackingGetIpcHandle + fun:umfMemoryProviderGetIPCHandle + fun:umfGetIPCHandle +} + +{ + [false-positive] trackingGetIpcHandle + drd:ConflictingAccess + fun:memmove + fun:trackingGetIpcHandle + fun:umfMemoryProviderGetIPCHandle + fun:umfGetIPCHandle +} diff --git a/test/supp/drd-test_provider_devdax_memory_ipc.supp b/test/supp/drd-test_provider_devdax_memory_ipc.supp index f6f12aa1ea..31608d30ca 100644 --- a/test/supp/drd-test_provider_devdax_memory_ipc.supp +++ b/test/supp/drd-test_provider_devdax_memory_ipc.supp @@ -2,6 +2,17 @@ [false-positive] Double check locking pattern in trackingOpenIpcHandle drd:ConflictingAccess fun:utils_atomic_store_release_ptr + fun:upstreamOpenIPCHandle + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + drd:ConflictingAccess + fun:utils_atomic_load_acquire_ptr fun:trackingOpenIpcHandle fun:umfMemoryProviderOpenIPCHandle fun:umfOpenIPCHandle diff --git a/test/supp/drd-test_provider_file_memory_ipc.supp b/test/supp/drd-test_provider_file_memory_ipc.supp index 72fd6d87cc..9883001f7c 100644 --- a/test/supp/drd-test_provider_file_memory_ipc.supp +++ b/test/supp/drd-test_provider_file_memory_ipc.supp @@ -10,6 +10,17 @@ [false-positive] Double check locking pattern in trackingOpenIpcHandle drd:ConflictingAccess fun:utils_atomic_store_release_ptr + fun:upstreamOpenIPCHandle + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + drd:ConflictingAccess + fun:utils_atomic_load_acquire_ptr fun:trackingOpenIpcHandle fun:umfMemoryProviderOpenIPCHandle fun:umfOpenIPCHandle diff --git a/test/supp/drd-test_provider_os_memory.supp b/test/supp/drd-test_provider_os_memory.supp index f6f12aa1ea..31608d30ca 100644 --- a/test/supp/drd-test_provider_os_memory.supp +++ b/test/supp/drd-test_provider_os_memory.supp @@ -2,6 +2,17 @@ [false-positive] Double check locking pattern in trackingOpenIpcHandle drd:ConflictingAccess fun:utils_atomic_store_release_ptr + fun:upstreamOpenIPCHandle + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + drd:ConflictingAccess + fun:utils_atomic_load_acquire_ptr fun:trackingOpenIpcHandle fun:umfMemoryProviderOpenIPCHandle fun:umfOpenIPCHandle diff --git a/test/supp/helgrind-test_ipc.supp b/test/supp/helgrind-test_ipc.supp index 04f3a91993..25ae87ea43 100644 --- a/test/supp/helgrind-test_ipc.supp +++ b/test/supp/helgrind-test_ipc.supp @@ -19,6 +19,7 @@ [false-positive] Double check locking pattern in trackingOpenIpcHandle Helgrind:Race fun:utils_atomic_store_release_ptr + fun:upstreamOpenIPCHandle fun:trackingOpenIpcHandle fun:umfMemoryProviderOpenIPCHandle fun:umfOpenIPCHandle diff --git a/test/supp/helgrind-test_ipc_max_opened_limit.supp b/test/supp/helgrind-test_ipc_max_opened_limit.supp new file mode 100644 index 0000000000..25ae87ea43 --- /dev/null +++ b/test/supp/helgrind-test_ipc_max_opened_limit.supp @@ -0,0 +1,54 @@ +{ + False-positive race in critnib_insert (lack of instrumentation) + Helgrind:Race + fun:utils_atomic_store_release_ptr + fun:critnib_insert + ... +} + +{ + False-positive race in critnib_find (lack of instrumentation) + Helgrind:Race + fun:find_predecessor + fun:find_le + fun:critnib_find + ... +} + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + Helgrind:Race + fun:utils_atomic_store_release_ptr + fun:upstreamOpenIPCHandle + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + Helgrind:Race + fun:utils_atomic_load_acquire_ptr + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} + +{ + [false-positive] umfMemoryProviderGetIPCHandle + Helgrind:Race + fun:trackingGetIpcHandle + fun:umfMemoryProviderGetIPCHandle + fun:umfGetIPCHandle +} + +{ + [false-positive] umfMemoryProviderGetIPCHandle + Helgrind:Race + fun:memmove + fun:trackingGetIpcHandle + fun:umfMemoryProviderGetIPCHandle + fun:umfGetIPCHandle +} diff --git a/test/supp/helgrind-test_provider_devdax_memory_ipc.supp b/test/supp/helgrind-test_provider_devdax_memory_ipc.supp index 4bc776f432..63e7d626c2 100644 --- a/test/supp/helgrind-test_provider_devdax_memory_ipc.supp +++ b/test/supp/helgrind-test_provider_devdax_memory_ipc.supp @@ -2,6 +2,17 @@ [false-positive] Double check locking pattern in trackingOpenIpcHandle Helgrind:Race fun:utils_atomic_store_release_ptr + fun:upstreamOpenIPCHandle + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + Helgrind:Race + fun:utils_atomic_load_acquire_ptr fun:trackingOpenIpcHandle fun:umfMemoryProviderOpenIPCHandle fun:umfOpenIPCHandle diff --git a/test/supp/helgrind-test_provider_file_memory_ipc.supp b/test/supp/helgrind-test_provider_file_memory_ipc.supp index de22665f51..11791e4ed2 100644 --- a/test/supp/helgrind-test_provider_file_memory_ipc.supp +++ b/test/supp/helgrind-test_provider_file_memory_ipc.supp @@ -2,6 +2,7 @@ [false-positive] Double check locking pattern in trackingOpenIpcHandle Helgrind:Race fun:utils_atomic_store_release_ptr + fun:upstreamOpenIPCHandle fun:trackingOpenIpcHandle fun:umfMemoryProviderOpenIPCHandle fun:umfOpenIPCHandle diff --git a/test/supp/helgrind-test_provider_os_memory.supp b/test/supp/helgrind-test_provider_os_memory.supp index 4bc776f432..63e7d626c2 100644 --- a/test/supp/helgrind-test_provider_os_memory.supp +++ b/test/supp/helgrind-test_provider_os_memory.supp @@ -2,6 +2,17 @@ [false-positive] Double check locking pattern in trackingOpenIpcHandle Helgrind:Race fun:utils_atomic_store_release_ptr + fun:upstreamOpenIPCHandle + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + Helgrind:Race + fun:utils_atomic_load_acquire_ptr fun:trackingOpenIpcHandle fun:umfMemoryProviderOpenIPCHandle fun:umfOpenIPCHandle diff --git a/test/test_valgrind.sh b/test/test_valgrind.sh index ea156e620b..2e4f655f64 100755 --- a/test/test_valgrind.sh +++ b/test/test_valgrind.sh @@ -188,11 +188,33 @@ echo echo "======================================================================" echo -for log in $(ls -1 ${PATH_TESTS}.log ${PATH_EXAMPLES}.log); do +LOG_FILES="" +NT=$(ls -1 ${PATH_TESTS}.log 2>/dev/null | wc -l) +if [ $NT -gt 0 ]; then + LOG_FILES="$LOG_FILES $(ls -1 ${PATH_TESTS}.log | xargs)" +fi +NE=$(ls -1 ${PATH_EXAMPLES}.log 2>/dev/null | wc -l) +if [ $NE -gt 0 ]; then + LOG_FILES="$LOG_FILES $(ls -1 ${PATH_EXAMPLES}.log | xargs)" +fi +if [ $(($NT + $NE)) -eq 0 ]; then + echo + echo "FATAL ERROR: no log files found, but number of failed tests equals $ANY_TEST_FAILED!" + echo + exit 1 +fi + +for log in $LOG_FILES; do echo ">>>>>>> LOG $log" cat $log echo echo done +if [ $(($NT + $NE)) -ne $ANY_TEST_FAILED ]; then + echo + echo "ERROR: incorrect number of log files: ANY_TEST_FAILED=$ANY_TEST_FAILED != ($NT + $NE)" + echo +fi + exit 1 diff --git a/src/cpp_helpers.hpp b/test/utils/cpp_helpers.hpp similarity index 97% rename from src/cpp_helpers.hpp rename to test/utils/cpp_helpers.hpp index 85e81c502b..037c633c17 100644 --- a/src/cpp_helpers.hpp +++ b/test/utils/cpp_helpers.hpp @@ -7,8 +7,8 @@ * */ -#ifndef UMF_HELPERS_HPP -#define UMF_HELPERS_HPP 1 +#ifndef UMF_TEST_HELPERS_HPP +#define UMF_TEST_HELPERS_HPP 1 #include #include @@ -22,7 +22,7 @@ #include #include -namespace umf { +namespace umf_test { using pool_unique_handle_t = std::unique_ptr umf_result_t &getPoolLastStatusRef() { return last_status; } -} // namespace umf +} // namespace umf_test -#endif /* UMF_HELPERS_HPP */ +#endif /* UMF_TEST_HELPERS_HPP */