From 90a2a9128b9d2e0c0ef79bc68cd60cae15156a8f Mon Sep 17 00:00:00 2001 From: Patryk Kaminski Date: Wed, 5 Mar 2025 13:01:54 +0100 Subject: [PATCH 01/47] Add lld linker CI job --- .github/workflows/reusable_basic.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 5a6756f2ca..41ce4b3850 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -74,6 +74,17 @@ jobs: install_tbb: 'ON' disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' + # test lld linker + - os: 'ubuntu-24.04' + build_type: Release + compiler: {c: icx, cxx: icpx} + shared_library: 'ON' + level_zero_provider: 'ON' + cuda_provider: 'ON' + install_tbb: 'ON' + disable_hwloc: 'OFF' + link_hwloc_statically: 'OFF' + llvm_linker: '-DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" -DCMAKE_MODULE_LINKER_FLAGS="-fuse-ld=lld" -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld"' # test without installing TBB - os: 'ubuntu-22.04' build_type: Release @@ -160,6 +171,7 @@ jobs: -DUMF_DISABLE_HWLOC=${{matrix.disable_hwloc}} -DUMF_LINK_HWLOC_STATICALLY=${{matrix.link_hwloc_statically}} ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' && '-DUMF_USE_COVERAGE=ON' || '' }} + ${{ matrix.llvm_linker || '' }} - name: Build UMF run: | From 45358d848ce4675563da5cdcb6d51cf916e1d9a0 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Thu, 6 Mar 2025 10:00:18 +0000 Subject: [PATCH 02/47] check for alloc fails in disjoint pool init --- src/pool/pool_disjoint.c | 50 ++++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/src/pool/pool_disjoint.c b/src/pool/pool_disjoint.c index 0bd88bd246..3855993330 100644 --- a/src/pool/pool_disjoint.c +++ b/src/pool/pool_disjoint.c @@ -588,12 +588,6 @@ umf_result_t disjoint_pool_initialize(umf_memory_provider_handle_t provider, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - disjoint_pool_t *disjoint_pool = - umf_ba_global_alloc(sizeof(*disjoint_pool)); - if (!disjoint_pool) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - umf_disjoint_pool_params_t *dp_params = (umf_disjoint_pool_params_t *)params; @@ -604,12 +598,21 @@ umf_result_t disjoint_pool_initialize(umf_memory_provider_handle_t provider, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } + disjoint_pool_t *disjoint_pool = + umf_ba_global_alloc(sizeof(*disjoint_pool)); + if (disjoint_pool == NULL) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + VALGRIND_DO_CREATE_MEMPOOL(disjoint_pool, 0, 0); disjoint_pool->provider = provider; disjoint_pool->params = *dp_params; disjoint_pool->known_slabs = critnib_new(); + if (disjoint_pool->known_slabs == NULL) { + goto err_free_disjoint_pool; + } // Generate buckets sized such as: 64, 96, 128, 192, ..., CutOff. // Powers of 2 and the value halfway between the powers of 2. @@ -625,6 +628,9 @@ umf_result_t disjoint_pool_initialize(umf_memory_provider_handle_t provider, disjoint_pool->min_bucket_size_exp = (size_t)log2Utils(Size1); disjoint_pool->default_shared_limits = umfDisjointPoolSharedLimitsCreate(SIZE_MAX); + if (disjoint_pool->default_shared_limits == NULL) { + goto err_free_known_slabs; + } // count number of buckets, start from 1 disjoint_pool->buckets_num = 1; @@ -633,10 +639,14 @@ umf_result_t disjoint_pool_initialize(umf_memory_provider_handle_t provider, for (; Size2 < CutOff; Size1 *= 2, Size2 *= 2) { disjoint_pool->buckets_num += 2; } + disjoint_pool->buckets = umf_ba_global_alloc( sizeof(*disjoint_pool->buckets) * disjoint_pool->buckets_num); + if (disjoint_pool->buckets == NULL) { + goto err_free_shared_limits; + } - int i = 0; + size_t i = 0; Size1 = ts1; Size2 = ts2; for (; Size2 < CutOff; Size1 *= 2, Size2 *= 2, i += 2) { @@ -648,6 +658,13 @@ umf_result_t disjoint_pool_initialize(umf_memory_provider_handle_t provider, disjoint_pool->buckets[i] = create_bucket( CutOff, disjoint_pool, disjoint_pool_get_limits(disjoint_pool)); + // check if all buckets were created successfully + for (i = 0; i < disjoint_pool->buckets_num; i++) { + if (disjoint_pool->buckets[i] == NULL) { + goto err_free_buckets; + } + } + umf_result_t ret = umfMemoryProviderGetMinPageSize( provider, NULL, &disjoint_pool->provider_min_page_size); if (ret != UMF_RESULT_SUCCESS) { @@ -657,6 +674,25 @@ umf_result_t disjoint_pool_initialize(umf_memory_provider_handle_t provider, *ppPool = (void *)disjoint_pool; return UMF_RESULT_SUCCESS; + +err_free_buckets: + for (i = 0; i < disjoint_pool->buckets_num; i++) { + if (disjoint_pool->buckets[i] != NULL) { + destroy_bucket(disjoint_pool->buckets[i]); + } + } + umf_ba_global_free(disjoint_pool->buckets); + +err_free_shared_limits: + umfDisjointPoolSharedLimitsDestroy(disjoint_pool->default_shared_limits); + +err_free_known_slabs: + critnib_delete(disjoint_pool->known_slabs); + +err_free_disjoint_pool: + umf_ba_global_free(disjoint_pool); + + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } void *disjoint_pool_malloc(void *pool, size_t size) { From 385fa4ce6d10b32b637433ac309276992f17586f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Thu, 6 Mar 2025 11:48:46 +0100 Subject: [PATCH 03/47] [CI] Add Compat test on GPU runners --- .github/workflows/reusable_compatibility.yml | 112 +++++++++++++++++-- 1 file changed, 103 insertions(+), 9 deletions(-) diff --git a/.github/workflows/reusable_compatibility.yml b/.github/workflows/reusable_compatibility.yml index 29597ac18f..a7b6311064 100644 --- a/.github/workflows/reusable_compatibility.yml +++ b/.github/workflows/reusable_compatibility.yml @@ -1,4 +1,4 @@ -# Workflow for checkig the backward compatibility of UMF. +# Workflow for checking the backward compatibility of UMF. # Test the latest UMF shared library with binaries compiled using the older UMF # shared library. name: Compatibility @@ -15,7 +15,7 @@ permissions: contents: read jobs: - ubuntu-build: + ubuntu: name: Ubuntu runs-on: 'ubuntu-22.04' @@ -64,7 +64,7 @@ jobs: working-directory: ${{github.workspace}}/tag_version run: | cmake --build ${{github.workspace}}/tag_version/build -j $(nproc) - + - name: Run "tag" UMF tests working-directory: ${{github.workspace}}/tag_version/build run: | @@ -75,13 +75,13 @@ jobs: with: fetch-depth: 0 path: ${{github.workspace}}/latest_version - + - name: Get latest UMF version working-directory: ${{github.workspace}}/latest_version run: | VERSION=$(git describe --tags) echo "checked version: $VERSION" - + - name: Configure latest UMF build working-directory: ${{github.workspace}}/latest_version run: > @@ -107,11 +107,11 @@ jobs: - name: Run "tag" UMF tests with latest UMF libs (warnings enabled) working-directory: ${{github.workspace}}/tag_version/build run: > - UMF_LOG="level:warning;flush:debug;output:stderr;pid:no" - LD_LIBRARY_PATH=${{github.workspace}}/latest_version/build/lib/ + UMF_LOG="level:warning;flush:debug;output:stderr;pid:no" + LD_LIBRARY_PATH=${{github.workspace}}/latest_version/build/lib/ ctest --output-on-failure - - windows-build: + + windows: name: Windows env: VCPKG_PATH: "${{github.workspace}}/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/vcpkg/packages/jemalloc_x64-windows" @@ -207,3 +207,97 @@ jobs: $env:UMF_LOG="level:warning;flush:debug;output:stderr;pid:no" cp ${{github.workspace}}/latest_version/build/bin/Debug/umf.dll ${{github.workspace}}/tag_version/build/bin/Debug/umf.dll ctest -C Debug --output-on-failure --test-dir test + + gpu: + name: GPU Ubuntu + strategy: + matrix: + provider: ['LEVEL_ZERO', 'CUDA'] + runs-on: ["DSS-${{matrix.provider}}", "DSS-UBUNTU"] + + steps: + - name: Checkout "tag" UMF version + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + ref: refs/tags/${{inputs.tag}} + path: ${{github.workspace}}/tag_version + + - name: Get "tag" UMF version + working-directory: ${{github.workspace}}/tag_version + run: | + VERSION=$(git describe --tags) + echo "tag version: $VERSION" + + - name: Configure "tag" UMF build + working-directory: ${{github.workspace}}/tag_version + run: > + cmake + -B ${{github.workspace}}/tag_version/build + -DCMAKE_BUILD_TYPE=Debug + -DUMF_BUILD_SHARED_LIBRARY=ON + -DCMAKE_C_COMPILER=gcc + -DCMAKE_CXX_COMPILER=g++ + -DUMF_BUILD_TESTS=ON + -DUMF_BUILD_GPU_TESTS=ON + -DUMF_BUILD_EXAMPLES=ON + -DUMF_BUILD_GPU_EXAMPLES=ON + -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF + -DUMF_BUILD_CUDA_PROVIDER=OFF + -DUMF_BUILD_${{matrix.provider}}_PROVIDER=ON + -DUMF_FORMAT_CODE_STYLE=OFF + -DUMF_DEVELOPER_MODE=ON + -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON + -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON + -DUMF_TESTS_FAIL_ON_SKIP=ON + + - name: Build "tag" UMF + working-directory: ${{github.workspace}}/tag_version + run: | + cmake --build ${{github.workspace}}/tag_version/build -j $(nproc) + + - name: Run "tag" UMF tests + working-directory: ${{github.workspace}}/tag_version/build + run: | + LD_LIBRARY_PATH=${{github.workspace}}/tag_version/build/lib/ ctest --output-on-failure + + - name: Checkout latest UMF version + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + path: ${{github.workspace}}/latest_version + + - name: Get latest UMF version + working-directory: ${{github.workspace}}/latest_version + run: | + VERSION=$(git describe --tags) + echo "checked version: $VERSION" + + - name: Configure latest UMF build + working-directory: ${{github.workspace}}/latest_version + run: > + cmake + -B ${{github.workspace}}/latest_version/build + -DCMAKE_BUILD_TYPE=Debug + -DUMF_BUILD_SHARED_LIBRARY=ON + -DCMAKE_C_COMPILER=gcc + -DCMAKE_CXX_COMPILER=g++ + -DUMF_BUILD_TESTS=OFF + -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON + -DUMF_BUILD_CUDA_PROVIDER=ON + -DUMF_FORMAT_CODE_STYLE=OFF + -DUMF_DEVELOPER_MODE=ON + -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON + + - name: Build latest UMF + working-directory: ${{github.workspace}}/latest_version + run: | + cmake --build ${{github.workspace}}/latest_version/build -j $(nproc) + + # NOTE: Once not implemented features may now be implemented - exclude these tests + - name: Run "tag" UMF tests with latest UMF libs (warnings enabled) + working-directory: ${{github.workspace}}/tag_version/build + run: > + UMF_LOG="level:warning;flush:debug;output:stderr;pid:no" + LD_LIBRARY_PATH=${{github.workspace}}/latest_version/build/lib/ + ctest --output-on-failure -E "not_impl" From 4760e50d217aa7ec80d1718f190beb5983548a3e Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Wed, 5 Mar 2025 16:54:44 +0100 Subject: [PATCH 04/47] [CI] Fix failing address sanitizer --- src/base_alloc/base_alloc_linux.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/base_alloc/base_alloc_linux.c b/src/base_alloc/base_alloc_linux.c index 260eec5aac..cc4f2e2bd6 100644 --- a/src/base_alloc/base_alloc_linux.c +++ b/src/base_alloc/base_alloc_linux.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -19,8 +19,8 @@ static UTIL_ONCE_FLAG Page_size_is_initialized = UTIL_ONCE_FLAG_INIT; static size_t Page_size; void *ba_os_alloc(size_t size) { - void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + void *ptr = utils_mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); // this should be unnecessary but pairs of mmap/munmap do not reset // asan's user-poisoning flags, leading to invalid error reports // Bug 81619: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81619 @@ -29,7 +29,7 @@ void *ba_os_alloc(size_t size) { } void ba_os_free(void *ptr, size_t size) { - int ret = munmap(ptr, size); + int ret = utils_munmap(ptr, size); assert(ret == 0); (void)ret; // unused } From cda4356f496915280cfa06345291cb7bf829496d Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Thu, 6 Mar 2025 15:13:10 +0100 Subject: [PATCH 05/47] Remove unnecessary headers from base alloc --- src/base_alloc/base_alloc_linux.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/base_alloc/base_alloc_linux.c b/src/base_alloc/base_alloc_linux.c index cc4f2e2bd6..9b1dc63fe0 100644 --- a/src/base_alloc/base_alloc_linux.c +++ b/src/base_alloc/base_alloc_linux.c @@ -6,13 +6,9 @@ */ #include -#include #include -#include #include -#include "base_alloc.h" -#include "base_alloc_global.h" #include "utils_concurrency.h" static UTIL_ONCE_FLAG Page_size_is_initialized = UTIL_ONCE_FLAG_INIT; From 1ef1e48dec5bb9349f4045c62854e5378633e7fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Thu, 6 Mar 2025 11:50:01 +0100 Subject: [PATCH 06/47] [CI][Compat] Remove steps printing version For tag checkout it's no surprise; for main we print full version during 'Configure' step. --- .github/workflows/reusable_compatibility.yml | 38 +------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/.github/workflows/reusable_compatibility.yml b/.github/workflows/reusable_compatibility.yml index a7b6311064..a11c91128b 100644 --- a/.github/workflows/reusable_compatibility.yml +++ b/.github/workflows/reusable_compatibility.yml @@ -35,12 +35,6 @@ jobs: - name: Install libhwloc working-directory: ${{github.workspace}}/tag_version run: .github/scripts/install_hwloc.sh - - - name: Get "tag" UMF version - working-directory: ${{github.workspace}}/tag_version - run: | - VERSION=$(git describe --tags) - echo "tag version: $VERSION" - name: Configure "tag" UMF build working-directory: ${{github.workspace}}/tag_version @@ -76,12 +70,6 @@ jobs: fetch-depth: 0 path: ${{github.workspace}}/latest_version - - name: Get latest UMF version - working-directory: ${{github.workspace}}/latest_version - run: | - VERSION=$(git describe --tags) - echo "checked version: $VERSION" - - name: Configure latest UMF build working-directory: ${{github.workspace}}/latest_version run: > @@ -132,16 +120,11 @@ jobs: vcpkgDirectory: ${{github.workspace}}/vcpkg vcpkgJsonGlob: '**/vcpkg.json' + # NOTE we use vcpkg setup from "tag" version - name: Install dependencies working-directory: ${{github.workspace}}/tag_version run: vcpkg install shell: pwsh # Specifies PowerShell as the shell for running the script. - - - name: Get "tag" UMF version - working-directory: ${{github.workspace}}/tag_version - run: | - $version = (git describe --tags) - echo "tag version: $VERSION" - name: Configure "tag" UMF build working-directory: ${{github.workspace}}/tag_version @@ -174,13 +157,6 @@ jobs: fetch-depth: 0 path: ${{github.workspace}}/latest_version - # NOTE we use vcpkg setup from "tag" version - - name: Get latest UMF version - working-directory: ${{github.workspace}}/latest_version - run: | - $version = (git describe --tags) - echo "latest version: $VERSION" - - name: Configure latest UMF build working-directory: ${{github.workspace}}/latest_version run: > @@ -223,12 +199,6 @@ jobs: ref: refs/tags/${{inputs.tag}} path: ${{github.workspace}}/tag_version - - name: Get "tag" UMF version - working-directory: ${{github.workspace}}/tag_version - run: | - VERSION=$(git describe --tags) - echo "tag version: $VERSION" - - name: Configure "tag" UMF build working-directory: ${{github.workspace}}/tag_version run: > @@ -267,12 +237,6 @@ jobs: fetch-depth: 0 path: ${{github.workspace}}/latest_version - - name: Get latest UMF version - working-directory: ${{github.workspace}}/latest_version - run: | - VERSION=$(git describe --tags) - echo "checked version: $VERSION" - - name: Configure latest UMF build working-directory: ${{github.workspace}}/latest_version run: > From f67374899f8cd3cad070c28c844e99cdee85aae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Thu, 6 Mar 2025 12:01:20 +0100 Subject: [PATCH 07/47] [CI][Compat] Run tests verbosely - warnings will be always visible --- .github/workflows/reusable_compatibility.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/reusable_compatibility.yml b/.github/workflows/reusable_compatibility.yml index a11c91128b..12444d6cfe 100644 --- a/.github/workflows/reusable_compatibility.yml +++ b/.github/workflows/reusable_compatibility.yml @@ -97,7 +97,7 @@ jobs: run: > UMF_LOG="level:warning;flush:debug;output:stderr;pid:no" LD_LIBRARY_PATH=${{github.workspace}}/latest_version/build/lib/ - ctest --output-on-failure + ctest --verbose windows: name: Windows @@ -182,7 +182,7 @@ jobs: run: | $env:UMF_LOG="level:warning;flush:debug;output:stderr;pid:no" cp ${{github.workspace}}/latest_version/build/bin/Debug/umf.dll ${{github.workspace}}/tag_version/build/bin/Debug/umf.dll - ctest -C Debug --output-on-failure --test-dir test + ctest -C Debug --verbose --test-dir test gpu: name: GPU Ubuntu @@ -264,4 +264,4 @@ jobs: run: > UMF_LOG="level:warning;flush:debug;output:stderr;pid:no" LD_LIBRARY_PATH=${{github.workspace}}/latest_version/build/lib/ - ctest --output-on-failure -E "not_impl" + ctest --verbose -E "not_impl" From f2a7e21aa286d15a2a147cb95549c6f3abeb623c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Thu, 6 Mar 2025 12:16:46 +0100 Subject: [PATCH 08/47] [CI][Compat] Enable examples as well --- .github/workflows/reusable_compatibility.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/reusable_compatibility.yml b/.github/workflows/reusable_compatibility.yml index 12444d6cfe..b6007b0812 100644 --- a/.github/workflows/reusable_compatibility.yml +++ b/.github/workflows/reusable_compatibility.yml @@ -46,6 +46,7 @@ jobs: -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DUMF_BUILD_TESTS=ON + -DUMF_BUILD_EXAMPLES=ON -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON -DUMF_BUILD_CUDA_PROVIDER=ON -DUMF_FORMAT_CODE_STYLE=OFF @@ -85,7 +86,6 @@ jobs: -DUMF_FORMAT_CODE_STYLE=OFF -DUMF_DEVELOPER_MODE=ON -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON - -DUMF_TESTS_FAIL_ON_SKIP=ON - name: Build latest UMF working-directory: ${{github.workspace}}/latest_version @@ -136,6 +136,7 @@ jobs: -DCMAKE_CXX_COMPILER=cl -DUMF_BUILD_SHARED_LIBRARY=ON -DUMF_BUILD_TESTS=ON + -DUMF_BUILD_EXAMPLES=ON -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON -DUMF_BUILD_CUDA_PROVIDER=ON -DUMF_FORMAT_CODE_STYLE=OFF @@ -172,7 +173,6 @@ jobs: -DUMF_FORMAT_CODE_STYLE=OFF -DUMF_DEVELOPER_MODE=ON -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON - -DUMF_TESTS_FAIL_ON_SKIP=ON - name: Build latest UMF run: cmake --build "${{github.workspace}}/latest_version/build" --config Debug -j $Env:NUMBER_OF_PROCESSORS @@ -182,7 +182,7 @@ jobs: run: | $env:UMF_LOG="level:warning;flush:debug;output:stderr;pid:no" cp ${{github.workspace}}/latest_version/build/bin/Debug/umf.dll ${{github.workspace}}/tag_version/build/bin/Debug/umf.dll - ctest -C Debug --verbose --test-dir test + ctest -C Debug --verbose gpu: name: GPU Ubuntu From cba0326c418982be7f1c822bdad7b070ee270a5f Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 7 Mar 2025 09:23:44 +0100 Subject: [PATCH 09/47] Add missing unlock in an error handling path of umf_ba_alloc() Signed-off-by: Lukasz Dorau --- src/base_alloc/base_alloc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/base_alloc/base_alloc.c b/src/base_alloc/base_alloc.c index 6f975307dc..00e58078e6 100644 --- a/src/base_alloc/base_alloc.c +++ b/src/base_alloc/base_alloc.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -230,6 +230,7 @@ void *umf_ba_alloc(umf_ba_pool_t *pool) { // check if the free list is not empty if (pool->metadata.free_list == NULL) { LOG_ERR("base_alloc: Free list should not be empty before new alloc"); + utils_mutex_unlock(&pool->metadata.free_lock); return NULL; } From 877dd1d38f463cb63bc98c322abe4f00a2499f8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Fri, 7 Mar 2025 10:03:59 +0100 Subject: [PATCH 10/47] Enable MT bench for disjoint_pool in fixed provider Ref. #1151 --- benchmark/benchmark.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index 7b04f2061e..377a38fcf3 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -165,9 +165,7 @@ UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, disjoint_pool_uniform_fixedprovider) ->Apply(&default_multiple_alloc_uniform_size) - ->Apply(&singlethreaded); -// TODO: change to multithreaded -//->Apply(&multithreaded); + ->Apply(&multithreaded); #ifdef UMF_POOL_JEMALLOC_ENABLED UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, From f9719a6984bff6559dcf6a6c6250779b83e1a0ef Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Wed, 5 Mar 2025 17:01:30 +0000 Subject: [PATCH 11/47] cleanup bit scan utils --- src/base_alloc/base_alloc_global.c | 6 ++-- src/critnib/critnib.c | 3 +- src/pool/pool_disjoint.c | 4 +-- src/utils/CMakeLists.txt | 7 ++--- src/utils/utils_concurrency.h | 15 --------- src/utils/utils_math.h | 50 +++++++++++++++++++++++++++--- src/utils/utils_posix_math.c | 20 ------------ src/utils/utils_windows_math.c | 24 -------------- 8 files changed, 56 insertions(+), 73 deletions(-) delete mode 100644 src/utils/utils_posix_math.c delete mode 100644 src/utils/utils_windows_math.c diff --git a/src/base_alloc/base_alloc_global.c b/src/base_alloc/base_alloc_global.c index f3b61566aa..ecec3367c9 100644 --- a/src/base_alloc/base_alloc_global.c +++ b/src/base_alloc/base_alloc_global.c @@ -71,7 +71,7 @@ static void umf_ba_create_global(void) { } size_t smallestSize = BASE_ALLOC.ac_sizes[0]; - BASE_ALLOC.smallest_ac_size_log2 = log2Utils(smallestSize); + BASE_ALLOC.smallest_ac_size_log2 = utils_msb64(smallestSize); LOG_DEBUG("UMF base allocator created"); } @@ -83,8 +83,8 @@ static int size_to_idx(size_t size) { } int isPowerOf2 = (0 == (size & (size - 1))); - int index = - (int)(log2Utils(size) + !isPowerOf2 - BASE_ALLOC.smallest_ac_size_log2); + int index = (int)(utils_msb64(size) + !isPowerOf2 - + BASE_ALLOC.smallest_ac_size_log2); assert(index >= 0); return index; diff --git a/src/critnib/critnib.c b/src/critnib/critnib.c index c95637f20c..1adb2dc7ee 100644 --- a/src/critnib/critnib.c +++ b/src/critnib/critnib.c @@ -64,6 +64,7 @@ #include "utils_assert.h" #include "utils_common.h" #include "utils_concurrency.h" +#include "utils_math.h" /* * A node that has been deleted is left untouched for this many delete @@ -367,7 +368,7 @@ int critnib_insert(struct critnib *c, word key, void *value, int update) { } /* and convert that to an index. */ - sh_t sh = utils_mssb_index(at) & (sh_t) ~(SLICE - 1); + sh_t sh = utils_msb64(at) & (sh_t) ~(SLICE - 1); struct critnib_node *m = alloc_node(c); if (!m) { diff --git a/src/pool/pool_disjoint.c b/src/pool/pool_disjoint.c index 0bd88bd246..82623988c7 100644 --- a/src/pool/pool_disjoint.c +++ b/src/pool/pool_disjoint.c @@ -466,7 +466,7 @@ static size_t size_to_idx(disjoint_pool_t *pool, size_t size) { } // get the position of the leftmost set bit - size_t position = getLeftmostSetBitPos(size); + size_t position = utils_msb64(size); bool is_power_of_2 = 0 == (size & (size - 1)); bool larger_than_halfway_between_powers_of_2 = @@ -622,7 +622,7 @@ umf_result_t disjoint_pool_initialize(umf_memory_provider_handle_t provider, Size1 = utils_max(Size1, UMF_DISJOINT_POOL_MIN_BUCKET_DEFAULT_SIZE); // Calculate the exponent for min_bucket_size used for finding buckets. - disjoint_pool->min_bucket_size_exp = (size_t)log2Utils(Size1); + disjoint_pool->min_bucket_size_exp = (size_t)utils_msb64(Size1); disjoint_pool->default_shared_limits = umfDisjointPoolSharedLimitsCreate(SIZE_MAX); diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index a0bff39fd8..976a2cb626 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (C) 2023-2024 Intel Corporation +# Copyright (C) 2023-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -7,15 +7,14 @@ include(FindThreads) set(UMF_UTILS_SOURCES_COMMON utils_common.c utils_log.c utils_load_library.c) -set(UMF_UTILS_SOURCES_POSIX utils_posix_common.c utils_posix_concurrency.c - utils_posix_math.c) +set(UMF_UTILS_SOURCES_POSIX utils_posix_common.c utils_posix_concurrency.c) set(UMF_UTILS_SOURCES_LINUX utils_linux_common.c) set(UMF_UTILS_SOURCES_MACOSX utils_macosx_common.c) set(UMF_UTILS_SOURCES_WINDOWS utils_windows_common.c - utils_windows_concurrency.c utils_windows_math.c) + utils_windows_concurrency.c) if(UMF_USE_VALGRIND) if(UMF_USE_ASAN diff --git a/src/utils/utils_concurrency.h b/src/utils/utils_concurrency.h index 0104b86468..e8a601ecd1 100644 --- a/src/utils/utils_concurrency.h +++ b/src/utils/utils_concurrency.h @@ -89,18 +89,6 @@ void utils_init_once(UTIL_ONCE_FLAG *flag, void (*onceCb)(void)); #if defined(_WIN32) -static inline unsigned char utils_lssb_index(long long value) { - unsigned long ret; - _BitScanForward64(&ret, value); - return (unsigned char)ret; -} - -static inline unsigned char utils_mssb_index(long long value) { - unsigned long ret; - _BitScanReverse64(&ret, value); - return (unsigned char)ret; -} - // There is no good way to do atomic_load on windows... static inline void utils_atomic_load_acquire_u64(uint64_t *ptr, uint64_t *out) { // NOTE: Windows cl complains about direct accessing 'ptr' which is next @@ -166,9 +154,6 @@ static inline bool utils_compare_exchange_u64(uint64_t *ptr, uint64_t *expected, #else // !defined(_WIN32) -#define utils_lssb_index(x) ((unsigned char)__builtin_ctzll(x)) -#define utils_mssb_index(x) ((unsigned char)(63 - __builtin_clzll(x))) - static inline void utils_atomic_load_acquire_u64(uint64_t *ptr, uint64_t *out) { ASSERT_IS_ALIGNED((uintptr_t)ptr, 8); ASSERT_IS_ALIGNED((uintptr_t)out, 8); diff --git a/src/utils/utils_math.h b/src/utils/utils_math.h index c78be11367..0e58fc38df 100644 --- a/src/utils/utils_math.h +++ b/src/utils/utils_math.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023-2024 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -11,16 +11,58 @@ #define UMF_MATH_H 1 #include +#include #include +#include #ifdef __cplusplus extern "C" { #endif -size_t getLeftmostSetBitPos(size_t num); +#if defined(_WIN32) -// Logarithm is an index of the most significant non-zero bit. -static inline size_t log2Utils(size_t num) { return getLeftmostSetBitPos(num); } +#include "utils_windows_intrin.h" + +#pragma intrinsic(_BitScanReverse64) +#pragma intrinsic(_BitScanForward64) + +// Retrieves the position of the leftmost set bit. +// The position of the bit is counted from 0 +// e.g. for 01000011110 the position equals 9. +static inline size_t utils_msb64(uint64_t num) { + assert(num != 0 && + "Finding leftmost set bit when number equals zero is undefined"); + unsigned long index = 0; + _BitScanReverse64(&index, num); + return (size_t)index; +} + +static inline size_t utils_lsb64(uint64_t num) { + assert(num != 0 && + "Finding rightmost set bit when number equals zero is undefined"); + unsigned long index = 0; + _BitScanForward64(&index, num); + return (size_t)index; +} + +#else // !defined(_WIN32) + +// Retrieves the position of the leftmost set bit. +// The position of the bit is counted from 0 +// e.g. for 01000011110 the position equals 9. +static inline size_t utils_msb64(uint64_t num) { + assert(num != 0 && + "Finding leftmost set bit when number equals zero is undefined"); + return 63 - __builtin_clzll(num); +} + +static inline size_t utils_lsb64(uint64_t num) { + assert(num != 0 && + "Finding rightmost set bit when number equals zero is undefined"); + return __builtin_ctzll(num); +} + +#endif // !defined(_WIN32) #ifdef __cplusplus } diff --git a/src/utils/utils_posix_math.c b/src/utils/utils_posix_math.c deleted file mode 100644 index 465b687725..0000000000 --- a/src/utils/utils_posix_math.c +++ /dev/null @@ -1,20 +0,0 @@ -/* - * - * Copyright (C) 2023 Intel Corporation - * - * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - * - */ - -#include "utils_math.h" -#include - -// Retrieves the position of the leftmost set bit. -// The position of the bit is counted from 0 -// e.g. for 01000011110 the position equals 9. -size_t getLeftmostSetBitPos(size_t num) { - assert(num != 0 && - "Finding leftmost set bit when number equals zero is undefined"); - return (sizeof(num) * CHAR_BIT - 1) - __builtin_clzll(num); -} diff --git a/src/utils/utils_windows_math.c b/src/utils/utils_windows_math.c deleted file mode 100644 index cd21ae696f..0000000000 --- a/src/utils/utils_windows_math.c +++ /dev/null @@ -1,24 +0,0 @@ -/* - * - * Copyright (C) 2023-2025 Intel Corporation - * - * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - * - */ - -#include "utils_math.h" -#include "utils_windows_intrin.h" - -#pragma intrinsic(_BitScanReverse) - -// Retrieves the position of the leftmost set bit. -// The position of the bit is counted from 0 -// e.g. for 01000011110 the position equals 9. -size_t getLeftmostSetBitPos(size_t num) { - assert(num != 0 && - "Finding leftmost set bit when number equals zero is undefined"); - unsigned long index = 0; - _BitScanReverse(&index, (unsigned long)num); - return (size_t)index; -} From d6930706a9171d7b9b0d35c6c19bb589f589ead0 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Wed, 5 Mar 2025 17:01:37 +0000 Subject: [PATCH 12/47] replace chunks bool array with bit fields --- src/pool/pool_disjoint.c | 71 +++++++++++++------------------ src/pool/pool_disjoint_internal.h | 35 ++++++++++++--- test/pools/disjoint_pool.cpp | 5 +-- 3 files changed, 61 insertions(+), 50 deletions(-) diff --git a/src/pool/pool_disjoint.c b/src/pool/pool_disjoint.c index 82623988c7..a380e09d8a 100644 --- a/src/pool/pool_disjoint.c +++ b/src/pool/pool_disjoint.c @@ -75,28 +75,36 @@ static slab_t *create_slab(bucket_t *bucket) { umf_result_t res = UMF_RESULT_SUCCESS; umf_memory_provider_handle_t provider = bucket->pool->provider; - slab_t *slab = umf_ba_global_alloc(sizeof(*slab)); + size_t num_chunks_total = + utils_max(bucket_slab_min_size(bucket) / bucket->size, 1); + + // Calculate the number of 64-bit words needed. + size_t num_words = + (num_chunks_total + CHUNK_BITMAP_SIZE - 1) / CHUNK_BITMAP_SIZE; + + slab_t *slab = umf_ba_global_alloc(sizeof(*slab) + + num_words * sizeof(slab->chunks[0])); if (slab == NULL) { LOG_ERR("allocation of new slab failed!"); return NULL; } slab->num_chunks_allocated = 0; - slab->first_free_chunk_idx = 0; slab->bucket = bucket; slab->iter.val = slab; slab->iter.prev = slab->iter.next = NULL; - slab->num_chunks_total = - utils_max(bucket_slab_min_size(bucket) / bucket->size, 1); - slab->chunks = - umf_ba_global_alloc(sizeof(*slab->chunks) * slab->num_chunks_total); - if (slab->chunks == NULL) { - LOG_ERR("allocation of slab chunks failed!"); - goto free_slab; + slab->num_chunks_total = num_chunks_total; + slab->num_words = num_words; + + // set all chunks as free + memset(slab->chunks, ~0, num_words * sizeof(slab->chunks[0])); + if (num_chunks_total % CHUNK_BITMAP_SIZE) { + // clear remaining bits + slab->chunks[num_words - 1] = + ((1ULL << (num_chunks_total % CHUNK_BITMAP_SIZE)) - 1); } - memset(slab->chunks, 0, sizeof(*slab->chunks) * slab->num_chunks_total); // if slab_min_size is not a multiple of bucket size, we would have some // padding at the end of the slab @@ -108,7 +116,7 @@ static slab_t *create_slab(bucket_t *bucket) { res = umfMemoryProviderAlloc(provider, slab->slab_size, 0, &slab->mem_ptr); if (res != UMF_RESULT_SUCCESS) { LOG_ERR("allocation of slab data failed!"); - goto free_slab_chunks; + goto free_slab; } // raw allocation is not available for user so mark it as inaccessible @@ -117,9 +125,6 @@ static slab_t *create_slab(bucket_t *bucket) { LOG_DEBUG("bucket: %p, slab_size: %zu", (void *)bucket, slab->slab_size); return slab; -free_slab_chunks: - umf_ba_global_free(slab->chunks); - free_slab: umf_ba_global_free(slab); return NULL; @@ -136,25 +141,21 @@ static void destroy_slab(slab_t *slab) { LOG_ERR("deallocation of slab data failed!"); } - umf_ba_global_free(slab->chunks); umf_ba_global_free(slab); } -// return the index of the first available chunk, SIZE_MAX otherwise static size_t slab_find_first_available_chunk_idx(const slab_t *slab) { - // use the first free chunk index as a hint for the search - for (bool *chunk = slab->chunks + slab->first_free_chunk_idx; - chunk != slab->chunks + slab->num_chunks_total; chunk++) { - - // false means not used - if (*chunk == false) { - size_t idx = chunk - slab->chunks; - LOG_DEBUG("idx: %zu", idx); - return idx; + for (size_t i = 0; i < slab->num_words; i++) { + // NOTE: free chunks are represented as set bits + uint64_t word = slab->chunks[i]; + if (word != 0) { + size_t bit_index = utils_lsb64(word); + size_t free_chunk = i * CHUNK_BITMAP_SIZE + bit_index; + return free_chunk; } } - LOG_DEBUG("idx: SIZE_MAX"); + // No free chunk was found. return SIZE_MAX; } @@ -167,12 +168,9 @@ static void *slab_get_chunk(slab_t *slab) { (void *)((uintptr_t)slab->mem_ptr + chunk_idx * slab->bucket->size); // mark chunk as used - slab->chunks[chunk_idx] = true; + slab_set_chunk_bit(slab, chunk_idx, false); slab->num_chunks_allocated += 1; - // use the found index as the next hint - slab->first_free_chunk_idx = chunk_idx + 1; - return free_chunk; } @@ -195,18 +193,9 @@ static void slab_free_chunk(slab_t *slab, void *ptr) { size_t chunk_idx = ptr_diff / slab->bucket->size; // Make sure that the chunk was allocated - assert(slab->chunks[chunk_idx] && "double free detected"); - slab->chunks[chunk_idx] = false; + assert(slab_read_chunk_bit(slab, chunk_idx) == 0 && "double free detected"); + slab_set_chunk_bit(slab, chunk_idx, true); slab->num_chunks_allocated -= 1; - - if (chunk_idx < slab->first_free_chunk_idx) { - slab->first_free_chunk_idx = chunk_idx; - } - - LOG_DEBUG("chunk_idx: %zu, num_chunks_allocated: %zu, " - "first_free_chunk_idx: %zu", - chunk_idx, slab->num_chunks_allocated, - slab->first_free_chunk_idx); } static bool slab_has_avail(const slab_t *slab) { diff --git a/src/pool/pool_disjoint_internal.h b/src/pool/pool_disjoint_internal.h index 2b5de64bc3..a930585fb7 100644 --- a/src/pool/pool_disjoint_internal.h +++ b/src/pool/pool_disjoint_internal.h @@ -15,6 +15,8 @@ #include "critnib/critnib.h" #include "utils_concurrency.h" +#define CHUNK_BITMAP_SIZE 64 + typedef struct bucket_t bucket_t; typedef struct slab_t slab_t; typedef struct slab_list_item_t slab_list_item_t; @@ -81,23 +83,24 @@ typedef struct slab_t { void *mem_ptr; size_t slab_size; - // Represents the current state of each chunk: if the bit is set, the - // chunk is allocated; otherwise, the chunk is free for allocation - bool *chunks; size_t num_chunks_total; + // Num of 64-bit words needed to store chunk state + size_t num_words; + // Total number of allocated chunks at the moment. size_t num_chunks_allocated; // The bucket which the slab belongs to bucket_t *bucket; - // Hints where to start search for free chunk in a slab - size_t first_free_chunk_idx; - // Store iterator to the corresponding node in avail/unavail list // to achieve O(1) removal slab_list_item_t iter; + + // Represents the current state of each chunk: if the bit is clear, the + // chunk is allocated; otherwise, the chunk is free for allocation + uint64_t chunks[]; } slab_t; typedef struct umf_disjoint_pool_shared_limits_t { @@ -158,4 +161,24 @@ typedef struct disjoint_pool_t { size_t provider_min_page_size; } disjoint_pool_t; +static inline void slab_set_chunk_bit(slab_t *slab, size_t index, bool value) { + assert(index < slab->num_chunks_total && "Index out of range"); + + size_t word_index = index / CHUNK_BITMAP_SIZE; + unsigned bit_index = index % CHUNK_BITMAP_SIZE; + if (value) { + slab->chunks[word_index] |= (1ULL << bit_index); + } else { + slab->chunks[word_index] &= ~(1ULL << bit_index); + } +} + +static inline int slab_read_chunk_bit(const slab_t *slab, size_t index) { + assert(index < slab->num_chunks_total && "Index out of range"); + + size_t word_index = index / CHUNK_BITMAP_SIZE; + unsigned bit_index = index % CHUNK_BITMAP_SIZE; + return (slab->chunks[word_index] >> bit_index) & 1; +} + #endif // UMF_POOL_DISJOINT_INTERNAL_H diff --git a/test/pools/disjoint_pool.cpp b/test/pools/disjoint_pool.cpp index 02f7698024..4eedce981d 100644 --- a/test/pools/disjoint_pool.cpp +++ b/test/pools/disjoint_pool.cpp @@ -113,9 +113,8 @@ TEST_F(test, internals) { EXPECT_GE(slab->num_chunks_total, slab->slab_size / bucket->size); // check allocation in slab - EXPECT_EQ(slab->chunks[0], true); - EXPECT_EQ(slab->chunks[1], false); - EXPECT_EQ(slab->first_free_chunk_idx, 1); + EXPECT_EQ(slab_read_chunk_bit(slab, 0), false); + EXPECT_EQ(slab_read_chunk_bit(slab, 1), true); // TODO: // * multiple alloc + free from single bucket From 0d5a89cffc2bcd45f72f17e69a01348c2354b1ad Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 10 Mar 2025 09:16:11 +0100 Subject: [PATCH 13/47] Run Compatibility GPU CI jobs only on upstream Run Compatibility GPU CI jobs only on upstream, since forks do not have the required HW. Signed-off-by: Lukasz Dorau --- .github/workflows/reusable_compatibility.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/reusable_compatibility.yml b/.github/workflows/reusable_compatibility.yml index b6007b0812..5bf9bd817b 100644 --- a/.github/workflows/reusable_compatibility.yml +++ b/.github/workflows/reusable_compatibility.yml @@ -186,6 +186,8 @@ jobs: gpu: name: GPU Ubuntu + # run only on upstream; forks will not have the HW + if: github.repository == 'oneapi-src/unified-memory-framework' strategy: matrix: provider: ['LEVEL_ZERO', 'CUDA'] From adce85f9c05d0230a1fba5dbac6686546b4bf286 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Mon, 3 Mar 2025 22:09:41 +0000 Subject: [PATCH 14/47] Use atomics in critnib find_* It fixes ThreadSanitizer data race in find_predecessor() vs critnib_insert() and critnib_remove(). Signed-off-by: Lukasz Dorau --- src/critnib/critnib.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/critnib/critnib.c b/src/critnib/critnib.c index 1adb2dc7ee..5c3a65dfdc 100644 --- a/src/critnib/critnib.c +++ b/src/critnib/critnib.c @@ -525,7 +525,9 @@ find_predecessor(struct critnib_node *__restrict n) { while (1) { int nib; for (nib = NIB; nib >= 0; nib--) { - if (n->child[nib]) { + struct critnib_node *m; + utils_atomic_load_acquire_ptr((void **)&n->child[nib], (void **)&m); + if (m) { break; } } @@ -534,7 +536,7 @@ find_predecessor(struct critnib_node *__restrict n) { return NULL; } - n = n->child[nib]; + utils_atomic_load_acquire_ptr((void **)&n->child[nib], (void **)&n); if (is_leaf(n)) { return to_leaf(n); } From 6483c5464774339335bdc8a8719fc77e7c2d119a Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 10 Mar 2025 09:03:36 +0100 Subject: [PATCH 15/47] Check valgrind log files Check valgrind log files. Do not print an error message like: ls: cannot access './examples/umf_example_*.log': No such file or directory when only the tests log files are present for example. Signed-off-by: Lukasz Dorau --- test/test_valgrind.sh | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/test/test_valgrind.sh b/test/test_valgrind.sh index ea156e620b..2e4f655f64 100755 --- a/test/test_valgrind.sh +++ b/test/test_valgrind.sh @@ -188,11 +188,33 @@ echo echo "======================================================================" echo -for log in $(ls -1 ${PATH_TESTS}.log ${PATH_EXAMPLES}.log); do +LOG_FILES="" +NT=$(ls -1 ${PATH_TESTS}.log 2>/dev/null | wc -l) +if [ $NT -gt 0 ]; then + LOG_FILES="$LOG_FILES $(ls -1 ${PATH_TESTS}.log | xargs)" +fi +NE=$(ls -1 ${PATH_EXAMPLES}.log 2>/dev/null | wc -l) +if [ $NE -gt 0 ]; then + LOG_FILES="$LOG_FILES $(ls -1 ${PATH_EXAMPLES}.log | xargs)" +fi +if [ $(($NT + $NE)) -eq 0 ]; then + echo + echo "FATAL ERROR: no log files found, but number of failed tests equals $ANY_TEST_FAILED!" + echo + exit 1 +fi + +for log in $LOG_FILES; do echo ">>>>>>> LOG $log" cat $log echo echo done +if [ $(($NT + $NE)) -ne $ANY_TEST_FAILED ]; then + echo + echo "ERROR: incorrect number of log files: ANY_TEST_FAILED=$ANY_TEST_FAILED != ($NT + $NE)" + echo +fi + exit 1 From 96d2ef39c110725f2f5e98a613c02c794bec4273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Plewa?= Date: Mon, 10 Mar 2025 13:56:14 +0100 Subject: [PATCH 16/47] add missing nullcheck in critnib --- src/critnib/critnib.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/critnib/critnib.c b/src/critnib/critnib.c index 5c3a65dfdc..feb492e206 100644 --- a/src/critnib/critnib.c +++ b/src/critnib/critnib.c @@ -537,6 +537,11 @@ find_predecessor(struct critnib_node *__restrict n) { } utils_atomic_load_acquire_ptr((void **)&n->child[nib], (void **)&n); + + if (!n) { + return NULL; + } + if (is_leaf(n)) { return to_leaf(n); } @@ -650,6 +655,11 @@ static struct critnib_leaf *find_successor(struct critnib_node *__restrict n) { } n = n->child[nib]; + + if (!n) { + return NULL; + } + if (is_leaf(n)) { return to_leaf(n); } From 8cc1d429c56b73545f783d67fdea6d2a3b8fc8f0 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 10 Mar 2025 14:22:47 +0100 Subject: [PATCH 17/47] Use atomics in find_successor() like in find_predecessor() Use atomics in find_successor() like in find_predecessor(). Ref: #1175 Signed-off-by: Lukasz Dorau --- src/critnib/critnib.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/critnib/critnib.c b/src/critnib/critnib.c index feb492e206..5625781d3f 100644 --- a/src/critnib/critnib.c +++ b/src/critnib/critnib.c @@ -645,7 +645,9 @@ static struct critnib_leaf *find_successor(struct critnib_node *__restrict n) { while (1) { unsigned nib; for (nib = 0; nib <= NIB; nib++) { - if (n->child[nib]) { + struct critnib_node *m; + utils_atomic_load_acquire_ptr((void **)&n->child[nib], (void **)&m); + if (m) { break; } } @@ -654,7 +656,7 @@ static struct critnib_leaf *find_successor(struct critnib_node *__restrict n) { return NULL; } - n = n->child[nib]; + utils_atomic_load_acquire_ptr((void **)&n->child[nib], (void **)&n); if (!n) { return NULL; From 965fc672d75f1d4e1be8534bd35063b940487902 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 4 Mar 2025 12:21:32 +0100 Subject: [PATCH 18/47] Add utils_atomic_store_release_u64() to utils Signed-off-by: Lukasz Dorau --- src/utils/utils_concurrency.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/utils/utils_concurrency.h b/src/utils/utils_concurrency.h index e8a601ecd1..31e5793b96 100644 --- a/src/utils/utils_concurrency.h +++ b/src/utils/utils_concurrency.h @@ -152,6 +152,17 @@ static inline bool utils_compare_exchange_u64(uint64_t *ptr, uint64_t *expected, return false; } +static inline void utils_atomic_store_release_u64(void *ptr, uint64_t val) { + ASSERT_IS_ALIGNED((uintptr_t)ptr, 8); + LONG64 out; + LONG64 desired = (LONG64)val; + LONG64 expected = 0; + while (expected != (out = InterlockedCompareExchange64( + (LONG64 volatile *)ptr, desired, expected))) { + expected = out; + } +} + #else // !defined(_WIN32) static inline void utils_atomic_load_acquire_u64(uint64_t *ptr, uint64_t *out) { @@ -168,6 +179,11 @@ static inline void utils_atomic_load_acquire_ptr(void **ptr, void **out) { utils_annotate_acquire(ptr); } +static inline void utils_atomic_store_release_u64(void *ptr, uint64_t val) { + ASSERT_IS_ALIGNED((uintptr_t)ptr, 8); + __atomic_store_n((uintptr_t *)ptr, (uintptr_t)val, memory_order_release); +} + static inline void utils_atomic_store_release_ptr(void **ptr, void *val) { ASSERT_IS_ALIGNED((uintptr_t)ptr, 8); utils_annotate_release(ptr); From d437d6c24581fe2fbd9f72891fa102281ff0533d Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 6 Mar 2025 09:43:51 +0100 Subject: [PATCH 19/47] Use eight level of critnibs in the tracker Multilevel maps are needed to support the case when one memory pool acts as a memory provider for another memory pool (nested memory pooling). Signed-off-by: Lukasz Dorau --- include/umf/base.h | 3 +- src/provider/provider_tracking.c | 430 +++++++++++++++++++++++++------ 2 files changed, 351 insertions(+), 82 deletions(-) diff --git a/include/umf/base.h b/include/umf/base.h index 8dad184f2b..12e99aa2bb 100644 --- a/include/umf/base.h +++ b/include/umf/base.h @@ -47,7 +47,8 @@ typedef enum umf_result_t { 6, ///< Failure in user provider code (i.e in user provided callback) UMF_RESULT_ERROR_DEPENDENCY_UNAVAILABLE = 7, ///< External required dependency is unavailable or missing - UMF_RESULT_ERROR_UNKNOWN = 0x7ffffffe ///< Unknown or internal error + UMF_RESULT_ERROR_OUT_OF_RESOURCES = 8, ///< Out of internal resources + UMF_RESULT_ERROR_UNKNOWN = 0x7ffffffe ///< Unknown error } umf_result_t; #ifdef __cplusplus diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index 4696bc562d..bc560304c7 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -7,73 +7,220 @@ * */ -#include "provider_tracking.h" +#include +#include +#include +#include +#include + +#include +#include +#include + #include "base_alloc_global.h" #include "critnib.h" #include "ipc_cache.h" #include "ipc_internal.h" +#include "provider_tracking.h" #include "utils_common.h" #include "utils_concurrency.h" #include "utils_log.h" -#include -#include -#include - -#include -#include -#include -#include -#include +// TODO: we need to support an arbitrary amount of layers in the future +#define MAX_LEVELS_OF_ALLOC_SEGMENT_MAP 8 uint64_t IPC_HANDLE_ID = 0; struct umf_memory_tracker_t { umf_ba_pool_t *alloc_info_allocator; - critnib *alloc_segments_map; + // Multilevel maps are needed to support the case + // when one memory pool acts as a memory provider + // for another memory pool (nested memory pooling). + critnib *alloc_segments_map[MAX_LEVELS_OF_ALLOC_SEGMENT_MAP]; utils_mutex_t splitMergeMutex; }; typedef struct tracker_alloc_info_t { umf_memory_pool_handle_t pool; size_t size; + // number of overlapping memory regions + // in the next level of map + // falling within the current range + size_t n_children; } tracker_alloc_info_t; -static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, - umf_memory_pool_handle_t pool, - const void *ptr, size_t size) { +// Get the most nested (on the highest level) allocation segment in the map with the `ptr` key. +// If `no_children` is set to 1, the function will return the entry +// only if it has no children on the higher level. +// The function returns the entry if found, otherwise NULL. +static tracker_alloc_info_t *get_most_nested_alloc_segment( + umf_memory_tracker_handle_t hTracker, const void *ptr, int *_level, + uintptr_t *_parent_key, tracker_alloc_info_t **_parent_value, + int no_children) { assert(ptr); + tracker_alloc_info_t *parent_value = NULL; + tracker_alloc_info_t *rvalue = NULL; + uintptr_t parent_key = 0; + uintptr_t rkey = 0; + uint64_t rsize = 0; + int level = 0; + int found = 0; + + do { + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + found = + critnib_find(hTracker->alloc_segments_map[level], (uintptr_t)ptr, + FIND_LE, (void *)&rkey, (void **)&rvalue); + if (!found || !rvalue) { + break; + } + + utils_atomic_load_acquire_u64((uint64_t *)&rvalue->size, &rsize); + + if (found && (uintptr_t)ptr < rkey + rsize) { + if (rvalue->n_children) { + if (level == MAX_LEVELS_OF_ALLOC_SEGMENT_MAP - 1) { + break; + } + level++; + parent_key = rkey; + parent_value = rvalue; + } + } + } while (found && ((uintptr_t)ptr < rkey + rsize) && rvalue->n_children); + + if (!rvalue || rkey != (uintptr_t)ptr) { + return NULL; + } + + if (no_children && (rvalue->n_children > 0)) { + return NULL; + } + + if (_level) { + *_level = level; + } + if (_parent_key) { + *_parent_key = parent_key; + } + if (_parent_value) { + *_parent_value = parent_value; + } + + assert(!no_children || rvalue->n_children == 0); + + return rvalue; +} + +static umf_result_t +umfMemoryTrackerAddAtLevel(umf_memory_tracker_handle_t hTracker, int level, + umf_memory_pool_handle_t pool, const void *ptr, + size_t size, uintptr_t parent_key, + tracker_alloc_info_t *parent_value) { + assert(ptr); + + umf_result_t umf_result = UMF_RESULT_ERROR_UNKNOWN; + tracker_alloc_info_t *value = umf_ba_alloc(hTracker->alloc_info_allocator); if (value == NULL) { - LOG_ERR("failed to allocate tracker value, ptr=%p, size=%zu", ptr, + LOG_ERR("failed to allocate a tracker value, ptr=%p, size=%zu", ptr, size); return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } value->pool = pool; - value->size = size; - - int ret = - critnib_insert(hTracker->alloc_segments_map, (uintptr_t)ptr, value, 0); + utils_atomic_store_release_u64(&value->size, size); + value->n_children = 0; + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + int ret = critnib_insert(hTracker->alloc_segments_map[level], + (uintptr_t)ptr, value, 0); if (ret == 0) { - LOG_DEBUG( - "memory region is added, tracker=%p, ptr=%p, pool=%p, size=%zu", - (void *)hTracker, ptr, (void *)pool, size); + LOG_DEBUG("memory region is added, tracker=%p, level=%i, pool=%p, " + "ptr=%p, size=%zu", + (void *)hTracker, level, (void *)pool, ptr, size); + + if (parent_value) { + parent_value->n_children++; + LOG_DEBUG( + "child #%zu added to memory region: tracker=%p, level=%i, " + "pool=%p, ptr=%p, size=%zu", + parent_value->n_children, (void *)hTracker, level - 1, + (void *)parent_value->pool, (void *)parent_key, + parent_value->size); + } return UMF_RESULT_SUCCESS; } + if (ret == ENOMEM) { + umf_result = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } - LOG_ERR("failed to insert tracker value, ret=%d, ptr=%p, pool=%p, size=%zu", - ret, ptr, (void *)pool, size); + LOG_ERR( + "failed to insert the tracker value: pool=%p, ptr=%p, size=%zu, ret=%d", + (void *)pool, ptr, size, ret); umf_ba_free(hTracker->alloc_info_allocator, value); - if (ret == ENOMEM) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + return umf_result; +} + +static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, + umf_memory_pool_handle_t pool, + const void *ptr, size_t size) { + assert(ptr); + + umf_result_t umf_result = UMF_RESULT_ERROR_UNKNOWN; + tracker_alloc_info_t *parent_value = NULL; + tracker_alloc_info_t *rvalue = NULL; + uintptr_t parent_key = 0; + uintptr_t rkey = 0; + uint64_t rsize = 0; + int level = 0; + int found = 0; + + // Find the most nested (in the highest level) entry + // in the critnib maps that contains the given 'ptr' pointer. + do { + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + found = + critnib_find(hTracker->alloc_segments_map[level], (uintptr_t)ptr, + FIND_LE, (void *)&rkey, (void **)&rvalue); + if (!found || !rvalue) { + break; + } + + utils_atomic_load_acquire_u64((uint64_t *)&rvalue->size, &rsize); + + if ((uintptr_t)ptr < rkey + rsize) { + if (level == MAX_LEVELS_OF_ALLOC_SEGMENT_MAP - 1) { + // TODO: we need to support an arbitrary amount of layers in the future + LOG_ERR("tracker level is too high, ptr=%p, size=%zu", ptr, + size); + return UMF_RESULT_ERROR_OUT_OF_RESOURCES; + } + if (((uintptr_t)ptr + size) > (rkey + rsize)) { + LOG_ERR( + "cannot insert to the tracker value (pool=%p, ptr=%p, " + "size=%zu) " + "that exceeds the parent value (pool=%p, ptr=%p, size=%zu)", + (void *)pool, ptr, size, (void *)rvalue->pool, (void *)rkey, + (size_t)rsize); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + parent_key = rkey; + parent_value = rvalue; + level++; + } + } while (found && ((uintptr_t)ptr < rkey + rsize) && rvalue->n_children); + + umf_result = umfMemoryTrackerAddAtLevel(hTracker, level, pool, ptr, size, + parent_key, parent_value); + if (umf_result != UMF_RESULT_SUCCESS) { + return umf_result; } - return UMF_RESULT_ERROR_UNKNOWN; + return UMF_RESULT_SUCCESS; } static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, @@ -85,16 +232,35 @@ static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, // Every umfMemoryTrackerAdd(..., ptr, ...) should have a corresponding // umfMemoryTrackerRemove call with the same ptr value. - void *value = critnib_remove(hTracker->alloc_segments_map, (uintptr_t)ptr); + tracker_alloc_info_t *parent_value = NULL; + uintptr_t parent_key = 0; + int level = 0; + + // Find the most nested (on the highest level) entry in the map + // with the `ptr` key and with no children - only such entry can be removed. + tracker_alloc_info_t *value = get_most_nested_alloc_segment( + hTracker, ptr, &level, &parent_key, &parent_value, 1 /* no_children */); if (!value) { LOG_ERR("pointer %p not found in the alloc_segments_map", ptr); return UMF_RESULT_ERROR_UNKNOWN; } - tracker_alloc_info_t *v = value; + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + value = critnib_remove(hTracker->alloc_segments_map[level], (uintptr_t)ptr); + assert(value); - LOG_DEBUG("memory region removed: tracker=%p, ptr=%p, size=%zu", - (void *)hTracker, ptr, v->size); + LOG_DEBUG("memory region removed: tracker=%p, level=%i, pool=%p, ptr=%p, " + "size=%zu", + (void *)hTracker, level, value->pool, ptr, value->size); + + if (parent_value) { + LOG_DEBUG( + "child #%zu removed from memory region: tracker=%p, level=%i, " + "pool=%p, ptr=%p, size=%zu", + parent_value->n_children, (void *)hTracker, level - 1, + (void *)parent_value->pool, (void *)parent_key, parent_value->size); + parent_value->n_children--; + } umf_ba_free(hTracker->alloc_info_allocator, value); @@ -124,24 +290,43 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, return UMF_RESULT_ERROR_NOT_SUPPORTED; } - if (TRACKER->alloc_segments_map == NULL) { + if (TRACKER->alloc_segments_map[0] == NULL) { LOG_ERR("tracker's alloc_segments_map does not exist"); return UMF_RESULT_ERROR_NOT_SUPPORTED; } - uintptr_t rkey; - tracker_alloc_info_t *rvalue; - int found = critnib_find(TRACKER->alloc_segments_map, (uintptr_t)ptr, + tracker_alloc_info_t *top_most_value = NULL; + tracker_alloc_info_t *rvalue = NULL; + uintptr_t top_most_key = 0; + uintptr_t rkey = 0; + int level = 0; + int found = 0; + + do { + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + found = critnib_find(TRACKER->alloc_segments_map[level], (uintptr_t)ptr, FIND_LE, (void *)&rkey, (void **)&rvalue); - if (!found || (uintptr_t)ptr >= rkey + rvalue->size) { + if (found && (uintptr_t)ptr < rkey + rvalue->size) { + top_most_key = rkey; + top_most_value = rvalue; + if (rvalue->n_children == 0 || + level == MAX_LEVELS_OF_ALLOC_SEGMENT_MAP - 1) { + break; + } + level++; + } + } while (found && (uintptr_t)ptr < rkey + rvalue->size && + rvalue->n_children); + + if (!top_most_value) { LOG_DEBUG("pointer %p not found in the tracker, TRACKER=%p", ptr, (void *)TRACKER); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - pAllocInfo->base = (void *)rkey; - pAllocInfo->baseSize = rvalue->size; - pAllocInfo->pool = rvalue->pool; + pAllocInfo->base = (void *)top_most_key; + pAllocInfo->baseSize = top_most_value->size; + pAllocInfo->pool = top_most_value->pool; return UMF_RESULT_SUCCESS; } @@ -166,26 +351,38 @@ typedef struct umf_tracking_memory_provider_t { typedef struct umf_tracking_memory_provider_t umf_tracking_memory_provider_t; static umf_result_t trackingAlloc(void *hProvider, size_t size, - size_t alignment, void **ptr) { + size_t alignment, void **_ptr) { umf_tracking_memory_provider_t *p = (umf_tracking_memory_provider_t *)hProvider; umf_result_t ret = UMF_RESULT_SUCCESS; + void *ptr; assert(p->hUpstream); - ret = umfMemoryProviderAlloc(p->hUpstream, size, alignment, ptr); - if (ret != UMF_RESULT_SUCCESS || !*ptr) { + *_ptr = NULL; + + ret = umfMemoryProviderAlloc(p->hUpstream, size, alignment, &ptr); + if (ret != UMF_RESULT_SUCCESS || !ptr) { return ret; } - umf_result_t ret2 = umfMemoryTrackerAdd(p->hTracker, p->pool, *ptr, size); - if (ret2 != UMF_RESULT_SUCCESS) { + ret = umfMemoryTrackerAdd(p->hTracker, p->pool, ptr, size); + if (ret != UMF_RESULT_SUCCESS) { LOG_ERR("failed to add allocated region to the tracker, ptr = %p, size " "= %zu, ret = %d", - *ptr, size, ret2); + ptr, size, ret); + umf_result_t ret2 = umfMemoryProviderFree(p->hUpstream, ptr, size); + if (ret2 != UMF_RESULT_SUCCESS) { + LOG_ERR("upstream provider failed to free the memory: ptr = %p, " + "size = %zu, ret = %d", + ptr, size, ret2); + } + return ret; } - return ret; + *_ptr = ptr; + + return UMF_RESULT_SUCCESS; } static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, @@ -194,6 +391,8 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, umf_result_t ret = UMF_RESULT_ERROR_UNKNOWN; umf_tracking_memory_provider_t *provider = (umf_tracking_memory_provider_t *)hProvider; + tracker_alloc_info_t *parent_value = NULL; + uintptr_t parent_key = 0; tracker_alloc_info_t *splitValue = umf_ba_alloc(provider->hTracker->alloc_info_allocator); @@ -203,21 +402,27 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, splitValue->pool = provider->pool; splitValue->size = firstSize; + splitValue->n_children = 0; int r = utils_mutex_lock(&provider->hTracker->splitMergeMutex); if (r) { goto err_lock; } - tracker_alloc_info_t *value = (tracker_alloc_info_t *)critnib_get( - provider->hTracker->alloc_segments_map, (uintptr_t)ptr); + int level = 0; + + // Find the most nested (on the highest level) entry in the map + // with the `ptr` key and with no children - only such entry can be split. + tracker_alloc_info_t *value = get_most_nested_alloc_segment( + provider->hTracker, ptr, &level, &parent_key, &parent_value, + 1 /* no_children */); if (!value) { LOG_ERR("region for split is not found in the tracker"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err; } if (value->size != totalSize) { - LOG_ERR("tracked size %zu does not match requested size to split: %zu", + LOG_ERR("tracked size=%zu does not match requested size to split: %zu", value->size, totalSize); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err; @@ -230,40 +435,58 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, goto err; } + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + int cret = + critnib_insert(provider->hTracker->alloc_segments_map[level], + (uintptr_t)ptr, (void *)splitValue, 1 /* update */); + // this cannot fail since we know the element exists (nothing to allocate) + assert(cret == 0); + (void)cret; + void *highPtr = (void *)(((uintptr_t)ptr) + firstSize); size_t secondSize = totalSize - firstSize; // We'll have a duplicate entry for the range [highPtr, highValue->size] but this is fine, // the value is the same anyway and we forbid removing that range concurrently - ret = umfMemoryTrackerAdd(provider->hTracker, provider->pool, highPtr, - secondSize); + ret = umfMemoryTrackerAddAtLevel(provider->hTracker, level, provider->pool, + highPtr, secondSize, parent_key, + parent_value); if (ret != UMF_RESULT_SUCCESS) { - LOG_ERR("failed to add split region to the tracker, ptr = %p, size " - "= %zu, ret = %d", + LOG_ERR("failed to add the split region to the tracker, ptr=%p, " + "size=%zu, ret=%d", highPtr, secondSize, ret); + // revert the split + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + cret = critnib_insert(provider->hTracker->alloc_segments_map[level], + (uintptr_t)ptr, (void *)value, 1 /* update */); + // this cannot fail since we know the element exists (nothing to allocate) + assert(cret == 0); + (void)cret; // TODO: what now? should we rollback the split? This can only happen due to ENOMEM // so it's unlikely but probably the best solution would be to try to preallocate everything // (value and critnib nodes) before calling umfMemoryProviderAllocationSplit. goto err; } - int cret = - critnib_insert(provider->hTracker->alloc_segments_map, (uintptr_t)ptr, - (void *)splitValue, 1 /* update */); - // this cannot fail since we know the element exists (nothing to allocate) - assert(cret == 0); - (void)cret; - // free the original value umf_ba_free(provider->hTracker->alloc_info_allocator, value); utils_mutex_unlock(&provider->hTracker->splitMergeMutex); + LOG_DEBUG( + "split memory region (level=%i): ptr=%p, totalSize=%zu, firstSize=%zu", + level, ptr, totalSize, firstSize); + return UMF_RESULT_SUCCESS; err: utils_mutex_unlock(&provider->hTracker->splitMergeMutex); err_lock: umf_ba_free(provider->hTracker->alloc_info_allocator, splitValue); + + LOG_ERR( + "failed to split memory region: ptr=%p, totalSize=%zu, firstSize=%zu", + ptr, totalSize, firstSize); + return ret; } @@ -282,26 +505,38 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, mergedValue->pool = provider->pool; mergedValue->size = totalSize; + mergedValue->n_children = 0; + + // any different negative values + int lowLevel = -2; + int highLevel = -1; int r = utils_mutex_lock(&provider->hTracker->splitMergeMutex); if (r) { goto err_lock; } - tracker_alloc_info_t *lowValue = (tracker_alloc_info_t *)critnib_get( - provider->hTracker->alloc_segments_map, (uintptr_t)lowPtr); + tracker_alloc_info_t *lowValue = get_most_nested_alloc_segment( + provider->hTracker, lowPtr, &lowLevel, NULL, NULL, + 0 /* no_children */); // can have children if (!lowValue) { LOG_FATAL("no left value"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err_assert; } - tracker_alloc_info_t *highValue = (tracker_alloc_info_t *)critnib_get( - provider->hTracker->alloc_segments_map, (uintptr_t)highPtr); + tracker_alloc_info_t *highValue = get_most_nested_alloc_segment( + provider->hTracker, highPtr, &highLevel, NULL, NULL, + 0 /* no_children */); // can have children if (!highValue) { LOG_FATAL("no right value"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err_assert; } + if (lowLevel != highLevel) { + LOG_FATAL("tracker level mismatch"); + ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; + goto err_assert; + } if (lowValue->pool != highValue->pool) { LOG_FATAL("pool mismatch"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; @@ -313,6 +548,8 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, goto err_assert; } + mergedValue->n_children = lowValue->n_children + highValue->n_children; + ret = umfMemoryProviderAllocationMerge(provider->hUpstream, lowPtr, highPtr, totalSize); if (ret != UMF_RESULT_SUCCESS) { @@ -320,10 +557,13 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, goto not_merged; } + size_t lno = lowValue->n_children; + size_t hno = highValue->n_children; + // We'll have a duplicate entry for the range [highPtr, highValue->size] but this is fine, // the value is the same anyway and we forbid removing that range concurrently int cret = - critnib_insert(provider->hTracker->alloc_segments_map, + critnib_insert(provider->hTracker->alloc_segments_map[lowLevel], (uintptr_t)lowPtr, (void *)mergedValue, 1 /* update */); // this cannot fail since we know the element exists (nothing to allocate) assert(cret == 0); @@ -333,16 +573,23 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, umf_ba_free(provider->hTracker->alloc_info_allocator, lowValue); void *erasedhighValue = critnib_remove( - provider->hTracker->alloc_segments_map, (uintptr_t)highPtr); + provider->hTracker->alloc_segments_map[highLevel], (uintptr_t)highPtr); assert(erasedhighValue == highValue); umf_ba_free(provider->hTracker->alloc_info_allocator, erasedhighValue); utils_mutex_unlock(&provider->hTracker->splitMergeMutex); + LOG_DEBUG("merged memory regions (level=%i): lowPtr=%p (child=%zu), " + "highPtr=%p (child=%zu), totalSize=%zu", + lowLevel, lowPtr, lno, highPtr, hno, totalSize); + return UMF_RESULT_SUCCESS; err_assert: + LOG_FATAL("failed to merge memory regions: lowPtr=%p (level=%i), " + "highPtr=%p (level=%i), totalSize=%zu", + lowPtr, lowLevel, highPtr, highLevel, totalSize); assert(0); not_merged: @@ -350,6 +597,11 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, err_lock: umf_ba_free(provider->hTracker->alloc_info_allocator, mergedValue); + + LOG_ERR("failed to merge memory regions: lowPtr=%p (level=%i), highPtr=%p " + "(level=%i), totalSize=%zu", + lowPtr, lowLevel, highPtr, highLevel, totalSize); + return ret; } @@ -428,19 +680,21 @@ static umf_result_t trackingInitialize(void *params, void **ret) { #ifndef NDEBUG static void check_if_tracker_is_empty(umf_memory_tracker_handle_t hTracker, umf_memory_pool_handle_t pool) { - uintptr_t rkey; - void *rvalue; size_t n_items = 0; - uintptr_t last_key = 0; - while (1 == critnib_find((critnib *)hTracker->alloc_segments_map, last_key, - FIND_G, &rkey, &rvalue)) { - tracker_alloc_info_t *value = (tracker_alloc_info_t *)rvalue; - if (value->pool == pool || pool == NULL) { - n_items++; - } + for (int i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { + uintptr_t last_key = 0; + uintptr_t rkey; + tracker_alloc_info_t *rvalue; + + while (1 == critnib_find(hTracker->alloc_segments_map[i], last_key, + FIND_G, &rkey, (void **)&rvalue)) { + if (rvalue->pool == pool || pool == NULL) { + n_items++; + } - last_key = rkey; + last_key = rkey; + } } if (n_items) { @@ -813,6 +1067,8 @@ umf_memory_tracker_handle_t umfMemoryTrackerCreate(void) { return NULL; } + memset(handle, 0, sizeof(struct umf_memory_tracker_t)); + umf_ba_pool_t *alloc_info_allocator = umf_ba_create(sizeof(struct tracker_alloc_info_t)); if (!alloc_info_allocator) { @@ -826,9 +1082,12 @@ umf_memory_tracker_handle_t umfMemoryTrackerCreate(void) { goto err_destroy_alloc_info_allocator; } - handle->alloc_segments_map = critnib_new(); - if (!handle->alloc_segments_map) { - goto err_destroy_mutex; + int i; + for (i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { + handle->alloc_segments_map[i] = critnib_new(); + if (!handle->alloc_segments_map[i]) { + goto err_destroy_mutex; + } } LOG_DEBUG("tracker created, handle=%p, alloc_segments_map=%p", @@ -837,6 +1096,11 @@ umf_memory_tracker_handle_t umfMemoryTrackerCreate(void) { return handle; err_destroy_mutex: + for (int j = i; j >= 0; j--) { + if (handle->alloc_segments_map[j]) { + critnib_delete(handle->alloc_segments_map[j]); + } + } utils_mutex_destroy_not_free(&handle->splitMergeMutex); err_destroy_alloc_info_allocator: umf_ba_destroy(alloc_info_allocator); @@ -864,8 +1128,12 @@ void umfMemoryTrackerDestroy(umf_memory_tracker_handle_t handle) { // We have to zero all inner pointers, // because the tracker handle can be copied // and used in many places. - critnib_delete(handle->alloc_segments_map); - handle->alloc_segments_map = NULL; + for (int i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { + if (handle->alloc_segments_map[i]) { + critnib_delete(handle->alloc_segments_map[i]); + handle->alloc_segments_map[i] = NULL; + } + } utils_mutex_destroy_not_free(&handle->splitMergeMutex); umf_ba_destroy(handle->alloc_info_allocator); handle->alloc_info_allocator = NULL; From 775ac129021e92bd04a2e385ba4cb2ee2cb553bb Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 25 Feb 2025 13:54:44 +0100 Subject: [PATCH 20/47] Add tests for pool from pointer to Fixed provider tests Signed-off-by: Lukasz Dorau --- test/provider_fixed_memory.cpp | 112 ++++++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/test/provider_fixed_memory.cpp b/test/provider_fixed_memory.cpp index 7f976a1f5d..1760ca4f74 100644 --- a/test/provider_fixed_memory.cpp +++ b/test/provider_fixed_memory.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2024 Intel Corporation +// Copyright (C) 2024-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -11,10 +11,12 @@ #endif #include +#include #include using umf_test::test; +#define FIXED_BUFFER_SIZE (10 * utils_get_page_size()) #define INVALID_PTR ((void *)0x01) typedef enum purge_t { @@ -59,7 +61,7 @@ struct FixedProviderTest test::SetUp(); // Allocate a memory buffer to use with the fixed memory provider - memory_size = utils_get_page_size() * 10; // Allocate 10 pages + memory_size = FIXED_BUFFER_SIZE; // Allocate 10 pages memory_buffer = malloc(memory_size); ASSERT_NE(memory_buffer, nullptr); @@ -391,3 +393,109 @@ TEST_P(FixedProviderTest, split) { umf_result = umfMemoryProviderFree(provider.get(), ptr2, size); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } + +TEST_P(FixedProviderTest, pool_from_ptr_whole_size_success) { + umf_result_t umf_result; + size_t size_of_first_alloc; + size_t size_of_pool_from_ptr; + void *ptr_for_pool = nullptr; + void *ptr = nullptr; + + umf_memory_pool_handle_t proxyFixedPool = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), provider.get(), nullptr, 0, + &proxyFixedPool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + size_of_first_alloc = FIXED_BUFFER_SIZE - (2 * page_size); + ptr_for_pool = umfPoolMalloc(proxyFixedPool, size_of_first_alloc); + ASSERT_NE(ptr_for_pool, nullptr); + + // Create provider parameters + size_of_pool_from_ptr = size_of_first_alloc; // whole size + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result = umfFixedMemoryProviderParamsCreate(¶ms, ptr_for_pool, + size_of_pool_from_ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + umf_memory_provider_handle_t providerFromPtr = nullptr; + umf_result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + &providerFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(providerFromPtr, nullptr); + + umf_memory_pool_handle_t poolFromPtr = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), providerFromPtr, nullptr, 0, + &poolFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ptr = umfPoolMalloc(poolFromPtr, size_of_pool_from_ptr); + ASSERT_NE(ptr, nullptr); + + memset(ptr, 0xFF, size_of_pool_from_ptr); + + umf_result = umfPoolFree(poolFromPtr, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(poolFromPtr); + umfMemoryProviderDestroy(providerFromPtr); + umfFixedMemoryProviderParamsDestroy(params); + + umf_result = umfPoolFree(proxyFixedPool, ptr_for_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(proxyFixedPool); +} + +TEST_P(FixedProviderTest, pool_from_ptr_half_size_success) { + umf_result_t umf_result; + size_t size_of_first_alloc; + size_t size_of_pool_from_ptr; + void *ptr_for_pool = nullptr; + void *ptr = nullptr; + + umf_memory_pool_handle_t proxyFixedPool = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), provider.get(), nullptr, 0, + &proxyFixedPool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + size_of_first_alloc = FIXED_BUFFER_SIZE - (2 * page_size); + ptr_for_pool = umfPoolMalloc(proxyFixedPool, size_of_first_alloc); + ASSERT_NE(ptr_for_pool, nullptr); + + // Create provider parameters + size_of_pool_from_ptr = size_of_first_alloc / 2; // half size + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result = umfFixedMemoryProviderParamsCreate(¶ms, ptr_for_pool, + size_of_pool_from_ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + umf_memory_provider_handle_t providerFromPtr = nullptr; + umf_result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + &providerFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(providerFromPtr, nullptr); + + umf_memory_pool_handle_t poolFromPtr = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), providerFromPtr, nullptr, 0, + &poolFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ptr = umfPoolMalloc(poolFromPtr, size_of_pool_from_ptr); + ASSERT_NE(ptr, nullptr); + + memset(ptr, 0xFF, size_of_pool_from_ptr); + + umf_result = umfPoolFree(poolFromPtr, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(poolFromPtr); + umfMemoryProviderDestroy(providerFromPtr); + umfFixedMemoryProviderParamsDestroy(params); + + umf_result = umfPoolFree(proxyFixedPool, ptr_for_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(proxyFixedPool); +} From cd61c1e6187966fede302fc98d33c2e6ca6e7449 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 25 Feb 2025 17:38:11 +0100 Subject: [PATCH 21/47] Add tests for pool from pointer to poolFixtures.hpp Signed-off-by: Lukasz Dorau --- test/poolFixtures.hpp | 149 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 144 insertions(+), 5 deletions(-) diff --git a/test/poolFixtures.hpp b/test/poolFixtures.hpp index 6f18664f92..6b01769f1b 100644 --- a/test/poolFixtures.hpp +++ b/test/poolFixtures.hpp @@ -5,11 +5,6 @@ #ifndef UMF_TEST_POOL_FIXTURES_HPP #define UMF_TEST_POOL_FIXTURES_HPP 1 -#include "pool.hpp" -#include "provider.hpp" -#include "umf/providers/provider_devdax_memory.h" -#include "utils/utils_sanitizers.h" - #include #include #include @@ -17,7 +12,14 @@ #include #include +#include +#include +#include + #include "../malloc_compliance_tests.hpp" +#include "pool.hpp" +#include "provider.hpp" +#include "utils/utils_sanitizers.h" typedef void *(*pfnPoolParamsCreate)(); typedef umf_result_t (*pfnPoolParamsDestroy)(void *); @@ -493,4 +495,141 @@ TEST_P(umfPoolTest, mallocUsableSize) { } } +TEST_P(umfPoolTest, umfPoolAlignedMalloc) { +#ifdef _WIN32 + // TODO: implement support for windows + GTEST_SKIP() << "umfPoolAlignedMalloc() is not supported on Windows"; +#else /* !_WIN32 */ + umf_result_t umf_result; + void *ptr = nullptr; + const size_t size = 2 * 1024 * 1024; // 2MB + + umf_memory_pool_handle_t pool_get = pool.get(); + + if (!umf_test::isAlignedAllocSupported(pool_get)) { + GTEST_SKIP(); + } + + ptr = umfPoolAlignedMalloc(pool_get, size, utils_get_page_size()); + ASSERT_NE(ptr, nullptr); + + umf_result = umfPoolFree(pool_get, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +#endif /* !_WIN32 */ +} + +TEST_P(umfPoolTest, pool_from_ptr_whole_size_success) { +#ifdef _WIN32 + // TODO: implement support for windows + GTEST_SKIP() << "umfPoolAlignedMalloc() is not supported on Windows"; +#else /* !_WIN32 */ + umf_result_t umf_result; + size_t size_of_pool_from_ptr; + void *ptr_for_pool = nullptr; + void *ptr = nullptr; + + umf_memory_pool_handle_t pool_get = pool.get(); + const size_t size_of_first_alloc = 2 * 1024 * 1024; // 2MB + + if (!umf_test::isAlignedAllocSupported(pool_get)) { + GTEST_SKIP(); + } + + ptr_for_pool = umfPoolAlignedMalloc(pool_get, size_of_first_alloc, + utils_get_page_size()); + ASSERT_NE(ptr_for_pool, nullptr); + + // Create provider parameters + size_of_pool_from_ptr = size_of_first_alloc; // whole size + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result = umfFixedMemoryProviderParamsCreate(¶ms, ptr_for_pool, + size_of_pool_from_ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + umf_memory_provider_handle_t providerFromPtr = nullptr; + umf_result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + &providerFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(providerFromPtr, nullptr); + + umf_memory_pool_handle_t poolFromPtr = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), providerFromPtr, nullptr, 0, + &poolFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ptr = umfPoolMalloc(poolFromPtr, size_of_pool_from_ptr); + ASSERT_NE(ptr, nullptr); + + memset(ptr, 0xFF, size_of_pool_from_ptr); + + umf_result = umfPoolFree(poolFromPtr, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(poolFromPtr); + umfMemoryProviderDestroy(providerFromPtr); + umfFixedMemoryProviderParamsDestroy(params); + + umf_result = umfPoolFree(pool_get, ptr_for_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +#endif /* !_WIN32 */ +} + +TEST_P(umfPoolTest, pool_from_ptr_half_size_success) { +#ifdef _WIN32 + // TODO: implement support for windows + GTEST_SKIP() << "umfPoolAlignedMalloc() is not supported on Windows"; +#else /* !_WIN32 */ + umf_result_t umf_result; + size_t size_of_pool_from_ptr; + void *ptr_for_pool = nullptr; + void *ptr = nullptr; + + umf_memory_pool_handle_t pool_get = pool.get(); + const size_t size_of_first_alloc = 2 * 1024 * 1024; // 2MB + + if (!umf_test::isAlignedAllocSupported(pool_get)) { + GTEST_SKIP(); + } + + ptr_for_pool = umfPoolAlignedMalloc(pool_get, size_of_first_alloc, + utils_get_page_size()); + ASSERT_NE(ptr_for_pool, nullptr); + + // Create provider parameters + size_of_pool_from_ptr = size_of_first_alloc / 2; // half size + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result = umfFixedMemoryProviderParamsCreate(¶ms, ptr_for_pool, + size_of_pool_from_ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + umf_memory_provider_handle_t providerFromPtr = nullptr; + umf_result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + &providerFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(providerFromPtr, nullptr); + + umf_memory_pool_handle_t poolFromPtr = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), providerFromPtr, nullptr, 0, + &poolFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ptr = umfPoolMalloc(poolFromPtr, size_of_pool_from_ptr); + ASSERT_NE(ptr, nullptr); + + memset(ptr, 0xFF, size_of_pool_from_ptr); + + umf_result = umfPoolFree(poolFromPtr, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(poolFromPtr); + umfMemoryProviderDestroy(providerFromPtr); + umfFixedMemoryProviderParamsDestroy(params); + + umf_result = umfPoolFree(pool_get, ptr_for_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +#endif /* !_WIN32 */ +} + #endif /* UMF_TEST_POOL_FIXTURES_HPP */ From bb4a5e48f9e903fddc1fa92d88db9a61c7ce2331 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 28 Feb 2025 16:43:39 +0100 Subject: [PATCH 22/47] Add tests for tracking provider Signed-off-by: Lukasz Dorau --- test/CMakeLists.txt | 4 + test/provider_tracking.cpp | 374 +++++++++++++++++++++++++++++++++++++ 2 files changed, 378 insertions(+) create mode 100644 test/provider_tracking.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 37f4c809ec..e47ce5a39d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -343,6 +343,10 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented NAME provider_fixed_memory SRCS provider_fixed_memory.cpp LIBS ${UMF_UTILS_FOR_TEST}) + add_umf_test( + NAME provider_tracking + SRCS provider_tracking.cpp + LIBS ${UMF_UTILS_FOR_TEST}) # This test requires Linux-only file memory provider if(UMF_POOL_JEMALLOC_ENABLED) diff --git a/test/provider_tracking.cpp b/test/provider_tracking.cpp new file mode 100644 index 0000000000..864c155649 --- /dev/null +++ b/test/provider_tracking.cpp @@ -0,0 +1,374 @@ +// Copyright (C) 2025 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "base.hpp" + +#include "cpp_helpers.hpp" +#include "test_helpers.h" +#ifndef _WIN32 +#include "test_helpers_linux.h" +#endif + +#include +#include +#include + +using umf_test::test; + +#define FIXED_BUFFER_SIZE (512 * utils_get_page_size()) +#define INVALID_PTR ((void *)0x01) + +using providerCreateExtParams = std::tuple; + +static void providerCreateExt(providerCreateExtParams params, + umf::provider_unique_handle_t *handle) { + umf_memory_provider_handle_t hProvider = nullptr; + auto [provider_ops, provider_params] = params; + + auto ret = + umfMemoryProviderCreate(provider_ops, provider_params, &hProvider); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(hProvider, nullptr); + + *handle = + umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); +} + +struct TrackingProviderTest + : umf_test::test, + ::testing::WithParamInterface { + void SetUp() override { + test::SetUp(); + + // Allocate a memory buffer to use with the fixed memory provider + memory_size = FIXED_BUFFER_SIZE; + memory_buffer = malloc(memory_size); + ASSERT_NE(memory_buffer, nullptr); + + // Create provider parameters + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result_t res = umfFixedMemoryProviderParamsCreate( + ¶ms, memory_buffer, memory_size); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + providerCreateExt(std::make_tuple(umfFixedMemoryProviderOps(), params), + &provider); + + umfFixedMemoryProviderParamsDestroy(params); + umf_result_t umf_result = + umfMemoryProviderGetMinPageSize(provider.get(), NULL, &page_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + page_plus_64 = page_size + 64; + + umf_memory_pool_handle_t hPool = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), provider.get(), nullptr, + 0, &hPool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + pool = umf::pool_unique_handle_t(hPool, &umfPoolDestroy); + } + + void TearDown() override { + if (memory_buffer) { + free(memory_buffer); + memory_buffer = nullptr; + } + test::TearDown(); + } + + umf::provider_unique_handle_t provider; + umf::pool_unique_handle_t pool; + size_t page_size; + size_t page_plus_64; + void *memory_buffer = nullptr; + size_t memory_size = 0; +}; + +static void +createPoolFromAllocation(void *ptr0, size_t size1, + umf_memory_provider_handle_t *_providerFromPtr, + umf_memory_pool_handle_t *_poolFromPtr) { + umf_result_t umf_result; + + // Create provider parameters + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result = umfFixedMemoryProviderParamsCreate(¶ms, ptr0, size1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + umf_memory_provider_handle_t provider1 = nullptr; + umf_result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + &provider1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider1, nullptr); + + umf_memory_pool_handle_t pool1 = nullptr; + umf_result = + umfPoolCreate(umfProxyPoolOps(), provider1, nullptr, 0, &pool1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfFixedMemoryProviderParamsDestroy(params); + + *_providerFromPtr = provider1; + *_poolFromPtr = pool1; +} + +// TESTS + +INSTANTIATE_TEST_SUITE_P(trackingProviderTest, TrackingProviderTest, + ::testing::Values(providerCreateExtParams{ + umfFixedMemoryProviderOps(), nullptr})); + +TEST_P(TrackingProviderTest, create_destroy) { + // Creation and destruction are handled in SetUp and TearDown +} + +TEST_P(TrackingProviderTest, whole_size_success) { + umf_result_t umf_result; + size_t size0; + size_t size1; + void *ptr0 = nullptr; + void *ptr1 = nullptr; + + umf_memory_pool_handle_t pool0 = pool.get(); + + size0 = FIXED_BUFFER_SIZE - (2 * page_size); + ptr0 = umfPoolAlignedMalloc(pool0, size0, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + size1 = size0; // whole size + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size1, &provider1, &pool1); + + ptr1 = umfPoolMalloc(pool1, size1); + ASSERT_NE(ptr1, nullptr); + + umf_result = umfPoolFree(pool1, ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(pool1); + umfMemoryProviderDestroy(provider1); + + umf_result = umfPoolFree(pool0, ptr0); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(TrackingProviderTest, half_size_success) { + umf_result_t umf_result; + size_t size0; + size_t size1; + void *ptr0 = nullptr; + void *ptr1 = nullptr; + + umf_memory_pool_handle_t pool0 = pool.get(); + + size0 = FIXED_BUFFER_SIZE - (2 * page_size); + ptr0 = umfPoolAlignedMalloc(pool0, size0, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + size1 = size0 / 2; // half size + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size1, &provider1, &pool1); + + ptr1 = umfPoolMalloc(pool1, size1); + ASSERT_NE(ptr1, nullptr); + + umf_result = umfPoolFree(pool1, ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(pool1); + umfMemoryProviderDestroy(provider1); + + umf_result = umfPoolFree(pool0, ptr0); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(TrackingProviderTest, failure_exceeding_size) { + umf_result_t umf_result; + size_t size0; + size_t size1; + void *ptr0 = nullptr; + void *ptr1 = nullptr; + + umf_memory_pool_handle_t pool0 = pool.get(); + + size0 = FIXED_BUFFER_SIZE - (2 * page_size); + ptr0 = umfPoolAlignedMalloc(pool0, size0, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + size1 = FIXED_BUFFER_SIZE - page_size; // exceeding size + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size1, &provider1, &pool1); + + ptr1 = umfPoolMalloc(pool1, size1); + ASSERT_EQ(ptr1, nullptr); + + umfPoolDestroy(pool1); + umfMemoryProviderDestroy(provider1); + + umf_result = umfPoolFree(pool0, ptr0); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +#define MAX_ARRAY 9 +#define TEST_LEVEL_SUCCESS 7 +#define TEST_LEVEL_FAILURE 8 + +TEST_P(TrackingProviderTest, success_max_levels) { + umf_result_t umf_result; + size_t size; + void *ptr[MAX_ARRAY] = {0}; + umf_memory_provider_handle_t providers[MAX_ARRAY] = {0}; + umf_memory_pool_handle_t pools[MAX_ARRAY] = {0}; + + size = FIXED_BUFFER_SIZE - (2 * page_size); + pools[0] = pool.get(); + + for (int i = 0; i < TEST_LEVEL_SUCCESS; i++) { + fprintf(stderr, "Alloc #%d\n", i); + ptr[i] = umfPoolAlignedMalloc(pools[i], size, utils_get_page_size()); + ASSERT_NE(ptr[i], nullptr); + + createPoolFromAllocation(ptr[i], size, &providers[i + 1], + &pools[i + 1]); + } + + int s = TEST_LEVEL_SUCCESS; + fprintf(stderr, "Alloc #%d\n", s); + ptr[s] = umfPoolAlignedMalloc(pools[s], size, utils_get_page_size()); + ASSERT_NE(ptr[s], nullptr); + + fprintf(stderr, "Free #%d\n", s); + umf_result = umfPoolFree(pools[s], ptr[s]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + for (int i = TEST_LEVEL_SUCCESS - 1; i >= 0; i--) { + umfPoolDestroy(pools[i + 1]); + umfMemoryProviderDestroy(providers[i + 1]); + + fprintf(stderr, "Free #%d\n", i); + umf_result = umfPoolFree(pools[i], ptr[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } +} + +TEST_P(TrackingProviderTest, failure_exceeding_levels) { + umf_result_t umf_result; + size_t size; + void *ptr[MAX_ARRAY] = {0}; + umf_memory_provider_handle_t providers[MAX_ARRAY] = {0}; + umf_memory_pool_handle_t pools[MAX_ARRAY] = {0}; + + size = FIXED_BUFFER_SIZE - (2 * page_size); + pools[0] = pool.get(); + + for (int i = 0; i < TEST_LEVEL_FAILURE; i++) { + fprintf(stderr, "Alloc #%d\n", i); + ptr[i] = umfPoolAlignedMalloc(pools[i], size, utils_get_page_size()); + ASSERT_NE(ptr[i], nullptr); + + createPoolFromAllocation(ptr[i], size, &providers[i + 1], + &pools[i + 1]); + } + + // tracker level is too high + int f = TEST_LEVEL_FAILURE; + fprintf(stderr, "Alloc #%d\n", f); + ptr[f] = umfPoolAlignedMalloc(pools[f], size, utils_get_page_size()); + ASSERT_EQ(ptr[f], nullptr); + + for (int i = TEST_LEVEL_FAILURE - 1; i >= 0; i--) { + umfPoolDestroy(pools[i + 1]); + umfMemoryProviderDestroy(providers[i + 1]); + + fprintf(stderr, "Free #%d\n", i); + umf_result = umfPoolFree(pools[i], ptr[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } +} + +TEST_P(TrackingProviderTest, reverted_free_half_size) { + umf_result_t umf_result; + size_t size0; + size_t size1; + void *ptr0 = nullptr; + void *ptr1 = nullptr; + + umf_memory_pool_handle_t pool0 = pool.get(); + + size0 = FIXED_BUFFER_SIZE - (2 * page_size); + ptr0 = umfPoolAlignedMalloc(pool0, size0, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size0, &provider1, &pool1); + + size1 = size0 / 2; // half size + + ptr1 = umfPoolMalloc(pool1, size1); + ASSERT_NE(ptr1, nullptr); + + // Freeing the "busy" pointer from the first pool is an Undefined Behavior + // It fails now if the sizes are different. + // see: https://github.com/oneapi-src/unified-memory-framework/pull/1161 + umf_result = umfPoolFree(pool0, ptr0); + + umf_result = umfPoolFree(pool1, ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(pool1); + umfMemoryProviderDestroy(provider1); + + // It could have been freed above, + // so we cannot verify the result here. + umf_result = umfPoolFree(pool0, ptr0); +} + +TEST_P(TrackingProviderTest, reverted_free_the_same_size) { + umf_result_t umf_result; + size_t size0; + size_t size1; + void *ptr0 = nullptr; + void *ptr1 = nullptr; + + umf_memory_pool_handle_t pool0 = pool.get(); + + size0 = FIXED_BUFFER_SIZE - (2 * page_size); + ptr0 = umfPoolAlignedMalloc(pool0, size0, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size0, &provider1, &pool1); + + size1 = size0; // the same size + + ptr1 = umfPoolMalloc(pool1, size1); + ASSERT_NE(ptr1, nullptr); + + // Freeing the "busy" pointer from the first pool is an Undefined Behavior + // It succeeds now if the sizes are equal. + // see: https://github.com/oneapi-src/unified-memory-framework/pull/1161 + umf_result = umfPoolFree(pool0, ptr0); + + // try to free the pointer from the second pool (the same size) + umf_result = umfPoolFree(pool1, ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(pool1); + umfMemoryProviderDestroy(provider1); + + // It could have been freed above, + // so we cannot verify the result here. + umf_result = umfPoolFree(pool0, ptr0); +} From c1b9f1bdc019d769a716f19b356b10c15ba22cdf Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 3 Mar 2025 12:34:48 +0100 Subject: [PATCH 23/47] Add provider_tracking_fixture_tests Signed-off-by: Lukasz Dorau --- test/CMakeLists.txt | 5 ++ test/provider_tracking_fixture_tests.cpp | 91 ++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 test/provider_tracking_fixture_tests.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e47ce5a39d..5f244b60e5 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -347,6 +347,11 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented NAME provider_tracking SRCS provider_tracking.cpp LIBS ${UMF_UTILS_FOR_TEST}) + add_umf_test( + NAME provider_tracking_fixture_tests + SRCS provider_tracking_fixture_tests.cpp malloc_compliance_tests.cpp + ${BA_SOURCES_FOR_TEST} + LIBS ${UMF_UTILS_FOR_TEST}) # This test requires Linux-only file memory provider if(UMF_POOL_JEMALLOC_ENABLED) diff --git a/test/provider_tracking_fixture_tests.cpp b/test/provider_tracking_fixture_tests.cpp new file mode 100644 index 0000000000..05b87f87fb --- /dev/null +++ b/test/provider_tracking_fixture_tests.cpp @@ -0,0 +1,91 @@ +// Copyright (C) 2025 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include + +#include "base.hpp" +#include "provider.hpp" + +#include "cpp_helpers.hpp" +#include "test_helpers.h" +#ifndef _WIN32 +#include "test_helpers_linux.h" +#endif + +#include "poolFixtures.hpp" + +#define FILE_PATH ((char *)"tmp_file") + +struct provider_from_pool : public umf_test::provider_base_t { + umf_memory_pool_handle_t pool; + umf_result_t initialize(umf_memory_pool_handle_t _pool) noexcept { + if (!_pool) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + pool = _pool; + return UMF_RESULT_SUCCESS; + } + umf_result_t alloc(size_t size, size_t align, void **ptr) noexcept { + *ptr = umfPoolAlignedMalloc(pool, size, align); + return (*ptr) ? UMF_RESULT_SUCCESS + : UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + umf_result_t free(void *ptr, size_t) noexcept { + return umfPoolFree(pool, ptr); + } + const char *get_name() noexcept { return "provider_from_pool"; } + + virtual ~provider_from_pool() { + if (pool) { + umfPoolDestroy(pool); + pool = nullptr; + } + } +}; + +umf_memory_provider_ops_t PROVIDER_FROM_POOL_OPS = + umf::providerMakeCOps(); + +static void *providerFromPoolParamsCreate(void) { + umf_file_memory_provider_params_handle_t paramsFile = NULL; + umf_result_t umf_result = + umfFileMemoryProviderParamsCreate(¶msFile, FILE_PATH); + EXPECT_EQ(umf_result, UMF_RESULT_SUCCESS); + EXPECT_NE(paramsFile, nullptr); + + umf_memory_provider_handle_t providerFile = nullptr; + umf_result = umfMemoryProviderCreate(umfFileMemoryProviderOps(), paramsFile, + &providerFile); + EXPECT_EQ(umf_result, UMF_RESULT_SUCCESS); + EXPECT_NE(providerFile, nullptr); + + umf_memory_pool_handle_t poolProxyFile = nullptr; + umf_result = + umfPoolCreate(umfProxyPoolOps(), providerFile, nullptr, + UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &poolProxyFile); + EXPECT_EQ(umf_result, UMF_RESULT_SUCCESS); + EXPECT_NE(poolProxyFile, nullptr); + + umf_result = umfFileMemoryProviderParamsDestroy(paramsFile); + EXPECT_EQ(umf_result, UMF_RESULT_SUCCESS); + paramsFile = nullptr; + + return poolProxyFile; +} + +// TESTS + +INSTANTIATE_TEST_SUITE_P(TrackingProviderPoolTest, umfPoolTest, + ::testing::Values(poolCreateExtParams{ + umfProxyPoolOps(), nullptr, nullptr, + &PROVIDER_FROM_POOL_OPS, + providerFromPoolParamsCreate, nullptr})); + +INSTANTIATE_TEST_SUITE_P(TrackingProviderMultiPoolTest, umfMultiPoolTest, + ::testing::Values(poolCreateExtParams{ + umfProxyPoolOps(), nullptr, nullptr, + &PROVIDER_FROM_POOL_OPS, + providerFromPoolParamsCreate, nullptr})); From 58007d625618b34a7ae36bfae799950a19d6b4be Mon Sep 17 00:00:00 2001 From: "Vinogradov, Sergei" Date: Tue, 17 Dec 2024 11:16:09 +0100 Subject: [PATCH 24/47] Implement size limit for the cache of opened IPC handles --- src/ipc.c | 14 +-- src/ipc_cache.c | 50 +++++++- src/ipc_cache.h | 2 + src/provider/provider_tracking.c | 210 ++++++++++++++++++++++++++----- src/provider/provider_tracking.h | 9 ++ test/ipcFixtures.hpp | 64 ---------- 6 files changed, 240 insertions(+), 109 deletions(-) diff --git a/src/ipc.c b/src/ipc.c index 12c7bb9786..d4e5cc8066 100644 --- a/src/ipc.c +++ b/src/ipc.c @@ -146,19 +146,15 @@ umf_result_t umfOpenIPCHandle(umf_ipc_handler_handle_t hIPCHandler, } umf_result_t umfCloseIPCHandle(void *ptr) { - umf_alloc_info_t allocInfo; - umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); + umf_ipc_info_t ipcInfo; + umf_result_t ret = umfMemoryTrackerGetIpcInfo(ptr, &ipcInfo); if (ret != UMF_RESULT_SUCCESS) { - LOG_ERR("cannot get alloc info for ptr = %p.", ptr); + LOG_ERR("cannot get IPC info for ptr = %p.", ptr); return ret; } - // We cannot use umfPoolGetMemoryProvider function because it returns - // upstream provider but we need tracking one - umf_memory_provider_handle_t hProvider = allocInfo.pool->provider; - - return umfMemoryProviderCloseIPCHandle(hProvider, allocInfo.base, - allocInfo.baseSize); + return umfMemoryProviderCloseIPCHandle(ipcInfo.provider, ipcInfo.base, + ipcInfo.baseSize); } umf_result_t umfPoolGetIPCHandler(umf_memory_pool_handle_t hPool, diff --git a/src/ipc_cache.c b/src/ipc_cache.c index 6d5d39e4f2..bf17a66a42 100644 --- a/src/ipc_cache.c +++ b/src/ipc_cache.c @@ -54,6 +54,22 @@ typedef struct ipc_opened_cache_t { ipc_opened_cache_global_t *IPC_OPENED_CACHE_GLOBAL = NULL; +// Returns value of the UMF_MAX_OPENED_IPC_HANDLES environment variable +// or 0 if it is not set. +static size_t umfIpcCacheGlobalInitMaxOpenedHandles(void) { + const char *max_size_str = getenv("UMF_MAX_OPENED_IPC_HANDLES"); + if (max_size_str) { + char *endptr; + size_t max_size = strtoul(max_size_str, &endptr, 10); + if (*endptr == '\0') { + return max_size; + } + LOG_ERR("Invalid value of UMF_MAX_OPENED_IPC_HANDLES: %s", + max_size_str); + } + return 0; +} + umf_result_t umfIpcCacheGlobalInit(void) { umf_result_t ret = UMF_RESULT_SUCCESS; ipc_opened_cache_global_t *cache_global = @@ -78,8 +94,7 @@ umf_result_t umfIpcCacheGlobalInit(void) { goto err_mutex_destroy; } - // TODO: make max_size configurable via environment variable - cache_global->max_size = 0; + cache_global->max_size = umfIpcCacheGlobalInitMaxOpenedHandles(); cache_global->cur_size = 0; cache_global->lru_list = NULL; @@ -191,7 +206,19 @@ umf_result_t umfIpcOpenedCacheGet(ipc_opened_cache_handle_t cache, if (entry == NULL && cache->global->max_size != 0 && cache->global->cur_size >= cache->global->max_size) { // If max_size is set and the cache is full, evict the least recently used entry. - entry = cache->global->lru_list->prev; + // we need to search for the least recently used entry with ref_count == 0 + // The utlist implementation of the doubly-linked list keeps a tail pointer in head->prev + ipc_opened_cache_entry_t *candidate = cache->global->lru_list->prev; + do { + uint64_t ref_count = 0; + utils_atomic_load_acquire_u64(&candidate->ref_count, + &ref_count); + if (ref_count == 0) { + entry = candidate; + break; + } + candidate = candidate->prev; + } while (candidate != cache->global->lru_list->prev); } if (entry) { // we have eviction candidate @@ -244,3 +271,20 @@ umf_result_t umfIpcOpenedCacheGet(ipc_opened_cache_handle_t cache, return ret; } + +umf_result_t +umfIpcHandleMappedCacheRelease(ipc_opened_cache_value_t *cacheValue) { + if (!cacheValue) { + LOG_ERR("cacheValue is NULL"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + // get pointer to the entry + ipc_opened_cache_entry_t *entry = + (ipc_opened_cache_entry_t *)((char *)cacheValue - + offsetof(ipc_opened_cache_entry_t, value)); + // decrement the ref count + utils_atomic_decrement_u64(&entry->ref_count); + + return UMF_RESULT_SUCCESS; +} diff --git a/src/ipc_cache.h b/src/ipc_cache.h index 80870d3737..545c6e1e7e 100644 --- a/src/ipc_cache.h +++ b/src/ipc_cache.h @@ -47,4 +47,6 @@ umf_result_t umfIpcOpenedCacheGet(ipc_opened_cache_handle_t cache, uint64_t handle_id, ipc_opened_cache_value_t **retEntry); +umf_result_t +umfIpcHandleMappedCacheRelease(ipc_opened_cache_value_t *cacheValue); #endif /* UMF_IPC_CACHE_H */ diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index bc560304c7..92d3dd59c6 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -21,6 +21,7 @@ #include "critnib.h" #include "ipc_cache.h" #include "ipc_internal.h" +#include "memory_pool_internal.h" #include "provider_tracking.h" #include "utils_common.h" #include "utils_concurrency.h" @@ -38,6 +39,8 @@ struct umf_memory_tracker_t { // for another memory pool (nested memory pooling). critnib *alloc_segments_map[MAX_LEVELS_OF_ALLOC_SEGMENT_MAP]; utils_mutex_t splitMergeMutex; + umf_ba_pool_t *ipc_info_allocator; + critnib *ipc_segments_map; }; typedef struct tracker_alloc_info_t { @@ -49,6 +52,12 @@ typedef struct tracker_alloc_info_t { size_t n_children; } tracker_alloc_info_t; +typedef struct tracker_ipc_info_t { + size_t size; + umf_memory_provider_handle_t provider; + ipc_opened_cache_value_t *ipc_cache_value; +} tracker_ipc_info_t; + // Get the most nested (on the highest level) allocation segment in the map with the `ptr` key. // If `no_children` is set to 1, the function will return the entry // only if it has no children on the higher level. @@ -267,6 +276,72 @@ static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, return UMF_RESULT_SUCCESS; } +static umf_result_t +umfMemoryTrackerAddIpcSegment(umf_memory_tracker_handle_t hTracker, + const void *ptr, size_t size, + umf_memory_provider_handle_t provider, + ipc_opened_cache_value_t *cache_entry) { + assert(hTracker); + assert(provider); + assert(cache_entry); + + tracker_ipc_info_t *value = umf_ba_alloc(hTracker->ipc_info_allocator); + + if (value == NULL) { + LOG_ERR("failed to allocate tracker_ipc_info_t, ptr=%p, size=%zu", ptr, + size); + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + value->size = size; + value->provider = provider; + value->ipc_cache_value = cache_entry; + + int ret = + critnib_insert(hTracker->ipc_segments_map, (uintptr_t)ptr, value, 0); + if (ret == 0) { + LOG_DEBUG("IPC memory region is added, tracker=%p, ptr=%p, size=%zu, " + "provider=%p, cache_entry=%p", + (void *)hTracker, ptr, size, provider, cache_entry); + return UMF_RESULT_SUCCESS; + } + + LOG_ERR("failed to insert tracker_ipc_info_t, ret=%d, ptr=%p, size=%zu, " + "provider=%p, cache_entry=%p", + ret, ptr, size, provider, cache_entry); + + umf_ba_free(hTracker->ipc_info_allocator, value); + + if (ret == ENOMEM) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + return UMF_RESULT_ERROR_UNKNOWN; +} + +static umf_result_t +umfMemoryTrackerRemoveIpcSegment(umf_memory_tracker_handle_t hTracker, + const void *ptr) { + assert(ptr); + + void *value = critnib_remove(hTracker->ipc_segments_map, (uintptr_t)ptr); + + if (!value) { + LOG_ERR("pointer %p not found in the ipc_segments_map", ptr); + return UMF_RESULT_ERROR_UNKNOWN; + } + + tracker_ipc_info_t *v = value; + + LOG_DEBUG("IPC memory region removed: tracker=%p, ptr=%p, size=%zu, " + "provider=%p, cache_entry=%p", + (void *)hTracker, ptr, v->size, v->provider, v->ipc_cache_value); + + umf_ba_free(hTracker->ipc_info_allocator, value); + + return UMF_RESULT_SUCCESS; +} + umf_memory_pool_handle_t umfMemoryTrackerGetPool(const void *ptr) { umf_alloc_info_t allocInfo = {NULL, 0, NULL}; umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); @@ -331,6 +406,41 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, return UMF_RESULT_SUCCESS; } +umf_result_t umfMemoryTrackerGetIpcInfo(const void *ptr, + umf_ipc_info_t *pIpcInfo) { + assert(pIpcInfo); + + if (ptr == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (TRACKER == NULL) { + LOG_ERR("tracker does not exist"); + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + if (TRACKER->ipc_segments_map == NULL) { + LOG_ERR("tracker's ipc_segments_map does not exist"); + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + uintptr_t rkey; + tracker_ipc_info_t *rvalue = NULL; + int found = critnib_find(TRACKER->ipc_segments_map, (uintptr_t)ptr, FIND_LE, + (void *)&rkey, (void **)&rvalue); + if (!found || (uintptr_t)ptr >= rkey + rvalue->size) { + LOG_DEBUG("pointer %p not found in the tracker, TRACKER=%p", ptr, + (void *)TRACKER); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + pIpcInfo->base = (void *)rkey; + pIpcInfo->baseSize = rvalue->size; + pIpcInfo->provider = rvalue->provider; + + return UMF_RESULT_SUCCESS; +} + // Cache entry structure to store provider-specific IPC data. // providerIpcData is a Flexible Array Member because its size varies // depending on the provider. @@ -872,17 +982,17 @@ ipcOpenedCacheEvictionCallback(const ipc_opened_cache_key_t *key, const ipc_opened_cache_value_t *value) { umf_tracking_memory_provider_t *p = (umf_tracking_memory_provider_t *)key->local_provider; - // umfMemoryTrackerRemove should be called before umfMemoryProviderCloseIPCHandle + // umfMemoryTrackerRemoveIpcSegment should be called before umfMemoryProviderCloseIPCHandle // to avoid a race condition. If the order would be different, other thread - // could allocate the memory at address `ptr` before a call to umfMemoryTrackerRemove + // could allocate the memory at address `ptr` before a call to umfMemoryTrackerRemoveIpcSegment // resulting in inconsistent state. if (value->mapped_base_ptr) { - umf_result_t ret = - umfMemoryTrackerRemove(p->hTracker, value->mapped_base_ptr); + umf_result_t ret = umfMemoryTrackerRemoveIpcSegment( + p->hTracker, value->mapped_base_ptr); if (ret != UMF_RESULT_SUCCESS) { // DO NOT return an error here, because the tracking provider // cannot change behaviour of the upstream provider. - LOG_ERR("failed to remove the region from the tracker, ptr=%p, " + LOG_ERR("failed to remove the region from the IPC tracker, ptr=%p, " "size=%zu, ret = %d", value->mapped_base_ptr, value->mapped_size, ret); } @@ -895,12 +1005,13 @@ ipcOpenedCacheEvictionCallback(const ipc_opened_cache_key_t *key, } } -static umf_result_t upstreamOpenIPCHandle(umf_tracking_memory_provider_t *p, - void *providerIpcData, - size_t bufferSize, void **ptr) { +static umf_result_t +upstreamOpenIPCHandle(umf_tracking_memory_provider_t *p, void *providerIpcData, + size_t bufferSize, + ipc_opened_cache_value_t *cache_entry) { void *mapped_ptr = NULL; assert(p != NULL); - assert(ptr != NULL); + assert(cache_entry != NULL); umf_result_t ret = umfMemoryProviderOpenIPCHandle( p->hUpstream, providerIpcData, &mapped_ptr); if (ret != UMF_RESULT_SUCCESS) { @@ -909,7 +1020,21 @@ static umf_result_t upstreamOpenIPCHandle(umf_tracking_memory_provider_t *p, } assert(mapped_ptr != NULL); - ret = umfMemoryTrackerAdd(p->hTracker, p->pool, mapped_ptr, bufferSize); + // Today umfMemoryTrackerAddIpcSegment requires the memory provider handle + // to know which tracking provider instance opened the IPC handle. + // The `p` points to the tracking provider private data. + // Because of that we get handle to the tracking provider instance + // using `p->pool->provider`. + // + // TODO: + // Today we always create a pool and get an IPC handler from the pool. + // And tracking provider is always created together with a pool. + // And the IPC handler is a tracking memory provider in fact. + // However, we are considering adding an API that allows IPC handler creation + // from scratch (without creating a memory pool). In that case, we will + // create a tracker provider without a pool. So p->pool might be NULL in the future. + ret = umfMemoryTrackerAddIpcSegment(p->hTracker, mapped_ptr, bufferSize, + p->pool->provider, cache_entry); if (ret != UMF_RESULT_SUCCESS) { LOG_ERR("failed to add IPC region to the tracker, ptr=%p, " "size=%zu, " @@ -924,7 +1049,8 @@ static umf_result_t upstreamOpenIPCHandle(umf_tracking_memory_provider_t *p, return ret; } - *ptr = mapped_ptr; + cache_entry->mapped_size = bufferSize; + utils_atomic_store_release_ptr(&(cache_entry->mapped_base_ptr), mapped_ptr); return UMF_RESULT_SUCCESS; } @@ -959,45 +1085,46 @@ static umf_result_t trackingOpenIpcHandle(void *provider, void *providerIpcData, void *mapped_ptr = NULL; utils_atomic_load_acquire_ptr(&(cache_entry->mapped_base_ptr), (void **)&mapped_ptr); - if (mapped_ptr == NULL) { + if (mapped_ptr == NULL) { // new cache entry utils_mutex_lock(&(cache_entry->mmap_lock)); utils_atomic_load_acquire_ptr(&(cache_entry->mapped_base_ptr), (void **)&mapped_ptr); if (mapped_ptr == NULL) { ret = upstreamOpenIPCHandle(p, providerIpcData, - ipcUmfData->baseSize, &mapped_ptr); - if (ret == UMF_RESULT_SUCCESS) { - // Put to the cache - cache_entry->mapped_size = ipcUmfData->baseSize; - utils_atomic_store_release_ptr(&(cache_entry->mapped_base_ptr), - mapped_ptr); - } + ipcUmfData->baseSize, cache_entry); } + mapped_ptr = cache_entry->mapped_base_ptr; utils_mutex_unlock(&(cache_entry->mmap_lock)); } if (ret == UMF_RESULT_SUCCESS) { + assert(mapped_ptr != NULL); *ptr = mapped_ptr; } return ret; } +static tracker_ipc_info_t *getTrackerIpcInfo(const void *ptr) { + assert(ptr); + + uintptr_t key = (uintptr_t)ptr; + tracker_ipc_info_t *value = critnib_get(TRACKER->ipc_segments_map, key); + + return value; +} + static umf_result_t trackingCloseIpcHandle(void *provider, void *ptr, size_t size) { (void)provider; - (void)ptr; - (void)size; - // We keep opened IPC handles in the p->hIpcMappedCache. - // IPC handle is closed when it is evicted from the cache - // or when cache is destroyed. - // - // TODO: today the size of the IPC cache is infinite. - // When the threshold for the cache size is implemented - // we need to introduce a reference counting mechanism. - // The trackingOpenIpcHandle will increment the refcount for the corresponding entry. - // The trackingCloseIpcHandle will decrement the refcount for the corresponding cache entry. - return UMF_RESULT_SUCCESS; + tracker_ipc_info_t *trackerIpcInfo = getTrackerIpcInfo(ptr); + + if (!trackerIpcInfo) { + LOG_ERR("failed to get tracker ipc info, ptr=%p, size=%zu", ptr, size); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + return umfIpcHandleMappedCacheRelease(trackerIpcInfo->ipc_cache_value); } umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { @@ -1086,16 +1213,29 @@ umf_memory_tracker_handle_t umfMemoryTrackerCreate(void) { for (i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { handle->alloc_segments_map[i] = critnib_new(); if (!handle->alloc_segments_map[i]) { - goto err_destroy_mutex; + goto err_destroy_alloc_segments_map; } } + handle->ipc_info_allocator = + umf_ba_create(sizeof(struct tracker_ipc_info_t)); + if (!handle->ipc_info_allocator) { + goto err_destroy_alloc_segments_map; + } + + handle->ipc_segments_map = critnib_new(); + if (!handle->ipc_segments_map) { + goto err_destroy_ipc_info_allocator; + } + LOG_DEBUG("tracker created, handle=%p, alloc_segments_map=%p", (void *)handle, (void *)handle->alloc_segments_map); return handle; -err_destroy_mutex: +err_destroy_ipc_info_allocator: + umf_ba_destroy(handle->ipc_info_allocator); +err_destroy_alloc_segments_map: for (int j = i; j >= 0; j--) { if (handle->alloc_segments_map[j]) { critnib_delete(handle->alloc_segments_map[j]); @@ -1137,5 +1277,9 @@ void umfMemoryTrackerDestroy(umf_memory_tracker_handle_t handle) { utils_mutex_destroy_not_free(&handle->splitMergeMutex); umf_ba_destroy(handle->alloc_info_allocator); handle->alloc_info_allocator = NULL; + critnib_delete(handle->ipc_segments_map); + handle->ipc_segments_map = NULL; + umf_ba_destroy(handle->ipc_info_allocator); + handle->ipc_info_allocator = NULL; umf_ba_global_free(handle); } diff --git a/src/provider/provider_tracking.h b/src/provider/provider_tracking.h index 9e868cf311..842449be5c 100644 --- a/src/provider/provider_tracking.h +++ b/src/provider/provider_tracking.h @@ -45,6 +45,15 @@ typedef struct umf_alloc_info_t { umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, umf_alloc_info_t *pAllocInfo); +typedef struct umf_ipc_info_t { + void *base; + size_t baseSize; + umf_memory_provider_handle_t provider; +} umf_ipc_info_t; + +umf_result_t umfMemoryTrackerGetIpcInfo(const void *ptr, + umf_ipc_info_t *pIpcInfo); + // Creates a memory provider that tracks each allocation/deallocation through umf_memory_tracker_handle_t and // forwards all requests to hUpstream memory Provider. hUpstream lifetime should be managed by the user of this function. umf_result_t umfTrackingMemoryProviderCreate( diff --git a/test/ipcFixtures.hpp b/test/ipcFixtures.hpp index 57bd040792..c898c3663a 100644 --- a/test/ipcFixtures.hpp +++ b/test/ipcFixtures.hpp @@ -389,70 +389,6 @@ TEST_P(umfIpcTest, BasicFlow) { EXPECT_EQ(stat.closeCount, stat.openCount); } -TEST_P(umfIpcTest, GetPoolByOpenedHandle) { - constexpr size_t SIZE = 100; - constexpr size_t NUM_ALLOCS = 100; - constexpr size_t NUM_POOLS = 4; - void *ptrs[NUM_ALLOCS]; - void *openedPtrs[NUM_POOLS][NUM_ALLOCS]; - std::vector pools_to_open; - umf::pool_unique_handle_t pool = makePool(); - ASSERT_NE(pool.get(), nullptr); - - for (size_t i = 0; i < NUM_POOLS; ++i) { - pools_to_open.push_back(makePool()); - } - - for (size_t i = 0; i < NUM_ALLOCS; ++i) { - void *ptr = umfPoolMalloc(pool.get(), SIZE); - ASSERT_NE(ptr, nullptr); - ptrs[i] = ptr; - } - - for (size_t i = 0; i < NUM_ALLOCS; ++i) { - umf_ipc_handle_t ipcHandle = nullptr; - size_t handleSize = 0; - umf_result_t ret = umfGetIPCHandle(ptrs[i], &ipcHandle, &handleSize); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - - for (size_t pool_id = 0; pool_id < NUM_POOLS; pool_id++) { - void *ptr = nullptr; - umf_ipc_handler_handle_t ipcHandler = nullptr; - ret = - umfPoolGetIPCHandler(pools_to_open[pool_id].get(), &ipcHandler); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - ASSERT_NE(ipcHandler, nullptr); - - ret = umfOpenIPCHandle(ipcHandler, ipcHandle, &ptr); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - openedPtrs[pool_id][i] = ptr; - } - - ret = umfPutIPCHandle(ipcHandle); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - } - - for (size_t pool_id = 0; pool_id < NUM_POOLS; pool_id++) { - for (size_t i = 0; i < NUM_ALLOCS; ++i) { - umf_memory_pool_handle_t openedPool = - umfPoolByPtr(openedPtrs[pool_id][i]); - EXPECT_EQ(openedPool, pools_to_open[pool_id].get()); - } - } - - for (size_t pool_id = 0; pool_id < NUM_POOLS; pool_id++) { - for (size_t i = 0; i < NUM_ALLOCS; ++i) { - umf_result_t ret = umfCloseIPCHandle(openedPtrs[pool_id][i]); - EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - } - } - - for (size_t i = 0; i < NUM_ALLOCS; ++i) { - umf_result_t ret = umfFree(ptrs[i]); - EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - } -} - TEST_P(umfIpcTest, AllocFreeAllocTest) { constexpr size_t SIZE = 64 * 1024; umf::pool_unique_handle_t pool = makePool(); From 02e38d7b0ca7ecc4a402313e92a5b29cd22a7dc1 Mon Sep 17 00:00:00 2001 From: Sergei Vinogradov Date: Mon, 17 Feb 2025 17:31:00 +0100 Subject: [PATCH 25/47] Add IPC test with UMF_MAX_OPENED_IPC_HANDLES set --- .cmake-format | 3 ++- test/CMakeLists.txt | 12 +++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.cmake-format b/.cmake-format index c1a8e85a83..f5f413d519 100644 --- a/.cmake-format +++ b/.cmake-format @@ -26,7 +26,8 @@ with section("parse"): 'kwargs': { 'NAME': '*', 'SRCS': '*', - 'LIBS': '*'}}, + 'LIBS': '*', + 'ENVS': '*'}}, 'add_umf_library': { "pargs": 0, "flags": [], diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5f244b60e5..e172115e16 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -116,8 +116,9 @@ function(add_umf_test) # * NAME - a name of the test # * SRCS - source files # * LIBS - libraries to be linked with + # * ENVS - environment variables set(oneValueArgs NAME) - set(multiValueArgs SRCS LIBS) + set(multiValueArgs SRCS LIBS ENVS) cmake_parse_arguments( ARG "" @@ -139,6 +140,9 @@ function(add_umf_test) WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_tests_properties(${TEST_NAME} PROPERTIES LABELS "umf") + if(ARG_ENVS) + set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT ${ARG_ENVS}) + endif() if(WINDOWS) # add PATH to DLL on Windows @@ -524,6 +528,12 @@ add_umf_test( SRCS ipcAPI.cpp ${BA_SOURCES_FOR_TEST} LIBS ${UMF_UTILS_FOR_TEST}) +add_umf_test( + NAME ipc_max_opened_limit + SRCS ipcAPI.cpp ${BA_SOURCES_FOR_TEST} + LIBS ${UMF_UTILS_FOR_TEST} + ENVS "UMF_MAX_OPENED_IPC_HANDLES=10") + add_umf_test(NAME ipc_negative SRCS ipc_negative.cpp) function(add_umf_ipc_test) From b40d71939104195ca2f0f2721890ba4349009ae3 Mon Sep 17 00:00:00 2001 From: Sergei Vinogradov Date: Fri, 21 Feb 2025 12:29:24 +0100 Subject: [PATCH 26/47] Add more IPC tests --- test/ipcFixtures.hpp | 241 +++++++++++++++++++++++++++++++------------ 1 file changed, 175 insertions(+), 66 deletions(-) diff --git a/test/ipcFixtures.hpp b/test/ipcFixtures.hpp index c898c3663a..1fc57b9002 100644 --- a/test/ipcFixtures.hpp +++ b/test/ipcFixtures.hpp @@ -68,6 +68,18 @@ using ipcTestParams = struct umfIpcTest : umf_test::test, ::testing::WithParamInterface { umfIpcTest() {} + size_t getOpenedIpcCacheSize() { + const char *max_size_str = getenv("UMF_MAX_OPENED_IPC_HANDLES"); + if (max_size_str) { + char *endptr; + size_t max_size = strtoul(max_size_str, &endptr, 10); + EXPECT_EQ(*endptr, '\0'); + if (*endptr == '\0') { + return max_size; + } + } + return 0; + } void SetUp() override { test::SetUp(); auto [pool_ops, pool_params_create, pool_params_destroy, provider_ops, @@ -80,6 +92,7 @@ struct umfIpcTest : umf_test::test, providerParamsCreate = provider_params_create; providerParamsDestroy = provider_params_destroy; memAccessor = accessor; + openedIpcCacheSize = getOpenedIpcCacheSize(); } void TearDown() override { test::TearDown(); } @@ -160,6 +173,7 @@ struct umfIpcTest : umf_test::test, umf_memory_provider_ops_t *providerOps = nullptr; pfnProviderParamsCreate providerParamsCreate = nullptr; pfnProviderParamsDestroy providerParamsDestroy = nullptr; + size_t openedIpcCacheSize = 0; void concurrentGetConcurrentPutHandles(bool shuffle) { std::vector ptrs; @@ -264,6 +278,156 @@ struct umfIpcTest : umf_test::test, pool.reset(nullptr); EXPECT_EQ(stat.putCount, stat.getCount); } + + void concurrentOpenConcurrentCloseHandles(bool shuffle) { + umf_result_t ret; + std::vector ptrs; + constexpr size_t ALLOC_SIZE = 100; + constexpr size_t NUM_POINTERS = 100; + umf::pool_unique_handle_t pool = makePool(); + ASSERT_NE(pool.get(), nullptr); + + for (size_t i = 0; i < NUM_POINTERS; ++i) { + void *ptr = umfPoolMalloc(pool.get(), ALLOC_SIZE); + EXPECT_NE(ptr, nullptr); + ptrs.push_back(ptr); + } + + std::vector ipcHandles; + for (size_t i = 0; i < NUM_POINTERS; ++i) { + umf_ipc_handle_t ipcHandle; + size_t handleSize; + ret = umfGetIPCHandle(ptrs[i], &ipcHandle, &handleSize); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ipcHandles.push_back(ipcHandle); + } + + std::array, NTHREADS> openedIpcHandles; + umf_ipc_handler_handle_t ipcHandler = nullptr; + ret = umfPoolGetIPCHandler(pool.get(), &ipcHandler); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(ipcHandler, nullptr); + + umf_test::syncthreads_barrier syncthreads(NTHREADS); + + auto openHandlesFn = [shuffle, &ipcHandles, &openedIpcHandles, + &syncthreads, ipcHandler](size_t tid) { + // Each thread gets a copy of the pointers to shuffle them + std::vector localIpcHandles = ipcHandles; + if (shuffle) { + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(localIpcHandles.begin(), localIpcHandles.end(), g); + } + syncthreads(); + for (auto ipcHandle : localIpcHandles) { + void *ptr; + umf_result_t ret = + umfOpenIPCHandle(ipcHandler, ipcHandle, &ptr); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + openedIpcHandles[tid].push_back(ptr); + } + }; + + umf_test::parallel_exec(NTHREADS, openHandlesFn); + + auto closeHandlesFn = [&openedIpcHandles, &syncthreads](size_t tid) { + syncthreads(); + for (void *ptr : openedIpcHandles[tid]) { + umf_result_t ret = umfCloseIPCHandle(ptr); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + } + }; + + umf_test::parallel_exec(NTHREADS, closeHandlesFn); + + for (auto ipcHandle : ipcHandles) { + ret = umfPutIPCHandle(ipcHandle); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + } + + for (void *ptr : ptrs) { + ret = umfPoolFree(pool.get(), ptr); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + } + + pool.reset(nullptr); + EXPECT_EQ(stat.getCount, stat.allocCount); + EXPECT_EQ(stat.putCount, stat.getCount); + EXPECT_EQ(stat.openCount, stat.allocCount); + EXPECT_EQ(stat.openCount, stat.closeCount); + } + + void concurrentOpenCloseHandles(bool shuffle) { + umf_result_t ret; + std::vector ptrs; + constexpr size_t ALLOC_SIZE = 100; + constexpr size_t NUM_POINTERS = 100; + umf::pool_unique_handle_t pool = makePool(); + ASSERT_NE(pool.get(), nullptr); + + for (size_t i = 0; i < NUM_POINTERS; ++i) { + void *ptr = umfPoolMalloc(pool.get(), ALLOC_SIZE); + EXPECT_NE(ptr, nullptr); + ptrs.push_back(ptr); + } + + std::vector ipcHandles; + for (size_t i = 0; i < NUM_POINTERS; ++i) { + umf_ipc_handle_t ipcHandle; + size_t handleSize; + ret = umfGetIPCHandle(ptrs[i], &ipcHandle, &handleSize); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ipcHandles.push_back(ipcHandle); + } + + umf_ipc_handler_handle_t ipcHandler = nullptr; + ret = umfPoolGetIPCHandler(pool.get(), &ipcHandler); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(ipcHandler, nullptr); + + umf_test::syncthreads_barrier syncthreads(NTHREADS); + + auto openCloseHandlesFn = [shuffle, &ipcHandles, &syncthreads, + ipcHandler](size_t) { + // Each thread gets a copy of the pointers to shuffle them + std::vector localIpcHandles = ipcHandles; + if (shuffle) { + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(localIpcHandles.begin(), localIpcHandles.end(), g); + } + syncthreads(); + for (auto ipcHandle : localIpcHandles) { + void *ptr; + umf_result_t ret = + umfOpenIPCHandle(ipcHandler, ipcHandle, &ptr); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ret = umfCloseIPCHandle(ptr); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + } + }; + + umf_test::parallel_exec(NTHREADS, openCloseHandlesFn); + + for (auto ipcHandle : ipcHandles) { + ret = umfPutIPCHandle(ipcHandle); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + } + + for (void *ptr : ptrs) { + ret = umfPoolFree(pool.get(), ptr); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + } + + pool.reset(nullptr); + EXPECT_EQ(stat.getCount, stat.allocCount); + EXPECT_EQ(stat.putCount, stat.getCount); + if (openedIpcCacheSize == 0) { + EXPECT_EQ(stat.openCount, stat.allocCount); + } + EXPECT_EQ(stat.openCount, stat.closeCount); + } }; TEST_P(umfIpcTest, GetIPCHandleSize) { @@ -529,75 +693,20 @@ TEST_P(umfIpcTest, ConcurrentGetPutHandlesShuffled) { concurrentGetPutHandles(true); } -TEST_P(umfIpcTest, ConcurrentOpenCloseHandles) { - umf_result_t ret; - std::vector ptrs; - constexpr size_t ALLOC_SIZE = 100; - constexpr size_t NUM_POINTERS = 100; - umf::pool_unique_handle_t pool = makePool(); - ASSERT_NE(pool.get(), nullptr); - - for (size_t i = 0; i < NUM_POINTERS; ++i) { - void *ptr = umfPoolMalloc(pool.get(), ALLOC_SIZE); - EXPECT_NE(ptr, nullptr); - ptrs.push_back(ptr); - } - - std::array ipcHandles; - for (size_t i = 0; i < NUM_POINTERS; ++i) { - umf_ipc_handle_t ipcHandle; - size_t handleSize; - ret = umfGetIPCHandle(ptrs[i], &ipcHandle, &handleSize); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - ipcHandles[i] = ipcHandle; - } - - std::array, NTHREADS> openedIpcHandles; - umf_ipc_handler_handle_t ipcHandler = nullptr; - ret = umfPoolGetIPCHandler(pool.get(), &ipcHandler); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - ASSERT_NE(ipcHandler, nullptr); - - umf_test::syncthreads_barrier syncthreads(NTHREADS); - - auto openHandlesFn = [&ipcHandles, &openedIpcHandles, &syncthreads, - ipcHandler](size_t tid) { - syncthreads(); - for (auto ipcHandle : ipcHandles) { - void *ptr; - umf_result_t ret = umfOpenIPCHandle(ipcHandler, ipcHandle, &ptr); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - openedIpcHandles[tid].push_back(ptr); - } - }; - - umf_test::parallel_exec(NTHREADS, openHandlesFn); - - auto closeHandlesFn = [&openedIpcHandles, &syncthreads](size_t tid) { - syncthreads(); - for (void *ptr : openedIpcHandles[tid]) { - umf_result_t ret = umfCloseIPCHandle(ptr); - EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - } - }; - - umf_test::parallel_exec(NTHREADS, closeHandlesFn); +TEST_P(umfIpcTest, ConcurrentOpenConcurrentCloseHandles) { + concurrentOpenConcurrentCloseHandles(false); +} - for (auto ipcHandle : ipcHandles) { - ret = umfPutIPCHandle(ipcHandle); - EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - } +TEST_P(umfIpcTest, ConcurrentOpenConcurrentCloseHandlesShuffled) { + concurrentOpenConcurrentCloseHandles(true); +} - for (void *ptr : ptrs) { - ret = umfPoolFree(pool.get(), ptr); - EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - } +TEST_P(umfIpcTest, ConcurrentOpenCloseHandles) { + concurrentOpenCloseHandles(false); +} - pool.reset(nullptr); - EXPECT_EQ(stat.getCount, stat.allocCount); - EXPECT_EQ(stat.putCount, stat.getCount); - EXPECT_EQ(stat.openCount, stat.allocCount); - EXPECT_EQ(stat.openCount, stat.closeCount); +TEST_P(umfIpcTest, ConcurrentOpenCloseHandlesShuffled) { + concurrentOpenCloseHandles(true); } TEST_P(umfIpcTest, ConcurrentDestroyIpcHandlers) { From c96de6190c012d1b738df68adaa4ce3951e90103 Mon Sep 17 00:00:00 2001 From: Sergei Vinogradov Date: Mon, 17 Feb 2025 22:11:43 +0100 Subject: [PATCH 27/47] Suppress drd and helgrind error in the umf_test-ipc_max_opened_limit --- test/supp/drd-test_ipc_max_opened_limit.supp | 34 ++++++++++++ .../drd-test_provider_devdax_memory_ipc.supp | 11 ++++ .../drd-test_provider_file_memory_ipc.supp | 11 ++++ test/supp/drd-test_provider_os_memory.supp | 11 ++++ .../helgrind-test_ipc_max_opened_limit.supp | 53 +++++++++++++++++++ ...grind-test_provider_devdax_memory_ipc.supp | 11 ++++ ...elgrind-test_provider_file_memory_ipc.supp | 1 + .../helgrind-test_provider_os_memory.supp | 11 ++++ 8 files changed, 143 insertions(+) create mode 100644 test/supp/drd-test_ipc_max_opened_limit.supp create mode 100644 test/supp/helgrind-test_ipc_max_opened_limit.supp diff --git a/test/supp/drd-test_ipc_max_opened_limit.supp b/test/supp/drd-test_ipc_max_opened_limit.supp new file mode 100644 index 0000000000..fbdbd0183f --- /dev/null +++ b/test/supp/drd-test_ipc_max_opened_limit.supp @@ -0,0 +1,34 @@ +{ + Conditional variable destruction false-positive + drd:CondErr + ... + fun:pthread_cond_destroy@* + ... +} + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + drd:ConflictingAccess + fun:utils_atomic_load_acquire_ptr + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} + +{ + [false-positive] trackingGetIpcHandle + drd:ConflictingAccess + fun:trackingGetIpcHandle + fun:umfMemoryProviderGetIPCHandle + fun:umfGetIPCHandle +} + +{ + [false-positive] trackingGetIpcHandle + drd:ConflictingAccess + fun:memmove + fun:trackingGetIpcHandle + fun:umfMemoryProviderGetIPCHandle + fun:umfGetIPCHandle +} diff --git a/test/supp/drd-test_provider_devdax_memory_ipc.supp b/test/supp/drd-test_provider_devdax_memory_ipc.supp index f6f12aa1ea..31608d30ca 100644 --- a/test/supp/drd-test_provider_devdax_memory_ipc.supp +++ b/test/supp/drd-test_provider_devdax_memory_ipc.supp @@ -2,6 +2,17 @@ [false-positive] Double check locking pattern in trackingOpenIpcHandle drd:ConflictingAccess fun:utils_atomic_store_release_ptr + fun:upstreamOpenIPCHandle + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + drd:ConflictingAccess + fun:utils_atomic_load_acquire_ptr fun:trackingOpenIpcHandle fun:umfMemoryProviderOpenIPCHandle fun:umfOpenIPCHandle diff --git a/test/supp/drd-test_provider_file_memory_ipc.supp b/test/supp/drd-test_provider_file_memory_ipc.supp index 72fd6d87cc..9883001f7c 100644 --- a/test/supp/drd-test_provider_file_memory_ipc.supp +++ b/test/supp/drd-test_provider_file_memory_ipc.supp @@ -10,6 +10,17 @@ [false-positive] Double check locking pattern in trackingOpenIpcHandle drd:ConflictingAccess fun:utils_atomic_store_release_ptr + fun:upstreamOpenIPCHandle + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + drd:ConflictingAccess + fun:utils_atomic_load_acquire_ptr fun:trackingOpenIpcHandle fun:umfMemoryProviderOpenIPCHandle fun:umfOpenIPCHandle diff --git a/test/supp/drd-test_provider_os_memory.supp b/test/supp/drd-test_provider_os_memory.supp index f6f12aa1ea..31608d30ca 100644 --- a/test/supp/drd-test_provider_os_memory.supp +++ b/test/supp/drd-test_provider_os_memory.supp @@ -2,6 +2,17 @@ [false-positive] Double check locking pattern in trackingOpenIpcHandle drd:ConflictingAccess fun:utils_atomic_store_release_ptr + fun:upstreamOpenIPCHandle + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + drd:ConflictingAccess + fun:utils_atomic_load_acquire_ptr fun:trackingOpenIpcHandle fun:umfMemoryProviderOpenIPCHandle fun:umfOpenIPCHandle diff --git a/test/supp/helgrind-test_ipc_max_opened_limit.supp b/test/supp/helgrind-test_ipc_max_opened_limit.supp new file mode 100644 index 0000000000..04f3a91993 --- /dev/null +++ b/test/supp/helgrind-test_ipc_max_opened_limit.supp @@ -0,0 +1,53 @@ +{ + False-positive race in critnib_insert (lack of instrumentation) + Helgrind:Race + fun:utils_atomic_store_release_ptr + fun:critnib_insert + ... +} + +{ + False-positive race in critnib_find (lack of instrumentation) + Helgrind:Race + fun:find_predecessor + fun:find_le + fun:critnib_find + ... +} + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + Helgrind:Race + fun:utils_atomic_store_release_ptr + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + Helgrind:Race + fun:utils_atomic_load_acquire_ptr + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} + +{ + [false-positive] umfMemoryProviderGetIPCHandle + Helgrind:Race + fun:trackingGetIpcHandle + fun:umfMemoryProviderGetIPCHandle + fun:umfGetIPCHandle +} + +{ + [false-positive] umfMemoryProviderGetIPCHandle + Helgrind:Race + fun:memmove + fun:trackingGetIpcHandle + fun:umfMemoryProviderGetIPCHandle + fun:umfGetIPCHandle +} diff --git a/test/supp/helgrind-test_provider_devdax_memory_ipc.supp b/test/supp/helgrind-test_provider_devdax_memory_ipc.supp index 4bc776f432..63e7d626c2 100644 --- a/test/supp/helgrind-test_provider_devdax_memory_ipc.supp +++ b/test/supp/helgrind-test_provider_devdax_memory_ipc.supp @@ -2,6 +2,17 @@ [false-positive] Double check locking pattern in trackingOpenIpcHandle Helgrind:Race fun:utils_atomic_store_release_ptr + fun:upstreamOpenIPCHandle + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + Helgrind:Race + fun:utils_atomic_load_acquire_ptr fun:trackingOpenIpcHandle fun:umfMemoryProviderOpenIPCHandle fun:umfOpenIPCHandle diff --git a/test/supp/helgrind-test_provider_file_memory_ipc.supp b/test/supp/helgrind-test_provider_file_memory_ipc.supp index de22665f51..11791e4ed2 100644 --- a/test/supp/helgrind-test_provider_file_memory_ipc.supp +++ b/test/supp/helgrind-test_provider_file_memory_ipc.supp @@ -2,6 +2,7 @@ [false-positive] Double check locking pattern in trackingOpenIpcHandle Helgrind:Race fun:utils_atomic_store_release_ptr + fun:upstreamOpenIPCHandle fun:trackingOpenIpcHandle fun:umfMemoryProviderOpenIPCHandle fun:umfOpenIPCHandle diff --git a/test/supp/helgrind-test_provider_os_memory.supp b/test/supp/helgrind-test_provider_os_memory.supp index 4bc776f432..63e7d626c2 100644 --- a/test/supp/helgrind-test_provider_os_memory.supp +++ b/test/supp/helgrind-test_provider_os_memory.supp @@ -2,6 +2,17 @@ [false-positive] Double check locking pattern in trackingOpenIpcHandle Helgrind:Race fun:utils_atomic_store_release_ptr + fun:upstreamOpenIPCHandle + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + Helgrind:Race + fun:utils_atomic_load_acquire_ptr fun:trackingOpenIpcHandle fun:umfMemoryProviderOpenIPCHandle fun:umfOpenIPCHandle From 72f02554ae1cc63c9646894f7d38195891ecdeee Mon Sep 17 00:00:00 2001 From: Sergei Vinogradov Date: Fri, 21 Feb 2025 14:12:47 +0100 Subject: [PATCH 28/47] Update docs about IPC caching --- docs/config/api.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/config/api.rst b/docs/config/api.rst index 1c20d709c2..97e664d97f 100644 --- a/docs/config/api.rst +++ b/docs/config/api.rst @@ -168,6 +168,26 @@ IPC API allows retrieving IPC handles for the memory buffers allocated from UMF memory pools. The memory provider used by the pool should support IPC operations for this API to work. Otherwise IPC APIs return an error. +IPC caching +------------------------------------------ + +UMF employs IPC caching to avoid multiple IPC handles being created for the same +coarse-grain memory region allocated by the memory provider. UMF guarantees that +for each coarse-grain memory region allocated by the memory provider, only one +IPC handle is created when the :any:`umfGetIPCHandle` function is called. All +subsequent calls to the :any:`umfGetIPCHandle` function for the pointer to the +same memory region will return the entry from the cache. + +The same is true for the :any:`umfOpenIPCHandle` function. The actual mapping +of the IPC handle to the virtual address space is created only once, and all +subsequent calls to open the same IPC handle will return the entry from the cache. +The size of the cache for opened IPC handles is controlled by the ``UMF_MAX_OPENED_IPC_HANDLES`` +environment variable. By default, the cache size is unlimited. However, if the environment +variable is set and the cache size exceeds the limit, old items will be evicted. UMF tracks +the ref count for each entry in the cache and can evict only items with the ref count equal to 0. +The ref count is increased when the :any:`umfOpenIPCHandle` function is called and decreased +when the :any:`umfCloseIPCHandle` function is called for the corresponding IPC handle. + .. _ipc-api: IPC API From 5f4336d33cbd4c1017f41ab38530e93b58c6a8e0 Mon Sep 17 00:00:00 2001 From: Sergei Vinogradov Date: Fri, 21 Feb 2025 19:18:54 +0100 Subject: [PATCH 29/47] Skip umfIpcTest.GetPoolByOpenedHandle test in compatibility testing --- .github/workflows/reusable_compatibility.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_compatibility.yml b/.github/workflows/reusable_compatibility.yml index 5bf9bd817b..5116a59f47 100644 --- a/.github/workflows/reusable_compatibility.yml +++ b/.github/workflows/reusable_compatibility.yml @@ -94,9 +94,11 @@ jobs: - name: Run "tag" UMF tests with latest UMF libs (warnings enabled) working-directory: ${{github.workspace}}/tag_version/build + # GTEST_FILTER is used below to skip test that is not compatible run: > UMF_LOG="level:warning;flush:debug;output:stderr;pid:no" LD_LIBRARY_PATH=${{github.workspace}}/latest_version/build/lib/ + GTEST_FILTER="-*umfIpcTest.GetPoolByOpenedHandle*" ctest --verbose windows: @@ -181,6 +183,7 @@ jobs: working-directory: ${{github.workspace}}/tag_version/build run: | $env:UMF_LOG="level:warning;flush:debug;output:stderr;pid:no" + $env:GTEST_FILTER="-*umfIpcTest.GetPoolByOpenedHandle*" cp ${{github.workspace}}/latest_version/build/bin/Debug/umf.dll ${{github.workspace}}/tag_version/build/bin/Debug/umf.dll ctest -C Debug --verbose @@ -230,8 +233,10 @@ jobs: - name: Run "tag" UMF tests working-directory: ${{github.workspace}}/tag_version/build - run: | - LD_LIBRARY_PATH=${{github.workspace}}/tag_version/build/lib/ ctest --output-on-failure + run: > + LD_LIBRARY_PATH=${{github.workspace}}/tag_version/build/lib/ + GTEST_FILTER="-*umfIpcTest.GetPoolByOpenedHandle*" + ctest --output-on-failure - name: Checkout latest UMF version uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -266,4 +271,5 @@ jobs: run: > UMF_LOG="level:warning;flush:debug;output:stderr;pid:no" LD_LIBRARY_PATH=${{github.workspace}}/latest_version/build/lib/ + GTEST_FILTER="-*umfIpcTest.GetPoolByOpenedHandle*" ctest --verbose -E "not_impl" From f0fa06fd91d670441a62c9865215b05a93182e06 Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Fri, 7 Mar 2025 16:36:52 +0100 Subject: [PATCH 30/47] [disjoint] Change pool name setting and cleanup --- include/umf/pools/pool_disjoint.h | 2 +- src/pool/pool_disjoint.c | 67 +++++++++++++------------------ src/pool/pool_disjoint_internal.h | 2 +- 3 files changed, 30 insertions(+), 41 deletions(-) diff --git a/include/umf/pools/pool_disjoint.h b/include/umf/pools/pool_disjoint.h index d268a1dac4..a1558b85bf 100644 --- a/include/umf/pools/pool_disjoint.h +++ b/include/umf/pools/pool_disjoint.h @@ -100,7 +100,7 @@ umf_result_t umfDisjointPoolParamsSetSharedLimits( /// @brief Set custom name of the disjoint pool to be used in the traces. /// @param hParams handle to the parameters of the disjoint pool. -/// @param name custom name of the pool. +/// @param name custom name of the pool. Name longer than 64 characters will be truncated. /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. umf_result_t umfDisjointPoolParamsSetName(umf_disjoint_pool_params_handle_t hParams, diff --git a/src/pool/pool_disjoint.c b/src/pool/pool_disjoint.c index 0bd88bd246..8661832f05 100644 --- a/src/pool/pool_disjoint.c +++ b/src/pool/pool_disjoint.c @@ -59,7 +59,7 @@ static __TLS umf_result_t TLS_last_allocation_error; // The largest size which is allocated via the allocator. // Allocations with size > CutOff bypass the pool and // go directly to the provider. -static size_t CutOff = (size_t)1 << 31; // 2GB +static const size_t CutOff = (size_t)1 << 31; // 2GB static size_t bucket_slab_min_size(bucket_t *bucket) { return bucket->pool->params.slab_min_size; @@ -468,7 +468,7 @@ static size_t size_to_idx(disjoint_pool_t *pool, size_t size) { // get the position of the leftmost set bit size_t position = getLeftmostSetBitPos(size); - bool is_power_of_2 = 0 == (size & (size - 1)); + bool is_power_of_2 = IS_POWER_OF_2(size); bool larger_than_halfway_between_powers_of_2 = !is_power_of_2 && (bool)((size - 1) & ((uint64_t)(1) << (position - 1))); @@ -630,8 +630,9 @@ umf_result_t disjoint_pool_initialize(umf_memory_provider_handle_t provider, disjoint_pool->buckets_num = 1; size_t Size2 = Size1 + Size1 / 2; size_t ts2 = Size2, ts1 = Size1; - for (; Size2 < CutOff; Size1 *= 2, Size2 *= 2) { + while (Size2 < CutOff) { disjoint_pool->buckets_num += 2; + Size2 *= 2; } disjoint_pool->buckets = umf_ba_global_alloc( sizeof(*disjoint_pool->buckets) * disjoint_pool->buckets_num); @@ -767,6 +768,14 @@ void *disjoint_pool_aligned_malloc(void *pool, size_t size, size_t alignment) { return aligned_ptr; } +static size_t get_chunk_idx(void *ptr, slab_t *slab) { + return (((uintptr_t)ptr - (uintptr_t)slab->mem_ptr) / slab->bucket->size); +} + +static void *get_unaligned_ptr(size_t chunk_idx, slab_t *slab) { + return (void *)((uintptr_t)slab->mem_ptr + chunk_idx * slab->bucket->size); +} + size_t disjoint_pool_malloc_usable_size(void *pool, void *ptr) { disjoint_pool_t *disjoint_pool = (disjoint_pool_t *)pool; if (ptr == NULL) { @@ -788,10 +797,8 @@ size_t disjoint_pool_malloc_usable_size(void *pool, void *ptr) { } // Get the unaligned pointer // NOTE: the base pointer slab->mem_ptr needn't to be aligned to bucket size - size_t chunk_idx = - (((uintptr_t)ptr - (uintptr_t)slab->mem_ptr) / slab->bucket->size); - void *unaligned_ptr = - (void *)((uintptr_t)slab->mem_ptr + chunk_idx * slab->bucket->size); + size_t chunk_idx = get_chunk_idx(ptr, slab); + void *unaligned_ptr = get_unaligned_ptr(chunk_idx, slab); ptrdiff_t diff = (ptrdiff_t)ptr - (ptrdiff_t)unaligned_ptr; @@ -847,10 +854,8 @@ umf_result_t disjoint_pool_free(void *pool, void *ptr) { // Get the unaligned pointer // NOTE: the base pointer slab->mem_ptr needn't to be aligned to bucket size - size_t chunk_idx = - (((uintptr_t)ptr - (uintptr_t)slab->mem_ptr) / slab->bucket->size); - void *unaligned_ptr = - (void *)((uintptr_t)slab->mem_ptr + chunk_idx * slab->bucket->size); + size_t chunk_idx = get_chunk_idx(ptr, slab); + void *unaligned_ptr = get_unaligned_ptr(chunk_idx, slab); utils_annotate_memory_inaccessible(unaligned_ptr, bucket->size); bucket_free_chunk(bucket, unaligned_ptr, slab, &to_pool); @@ -876,13 +881,11 @@ umf_result_t disjoint_pool_free(void *pool, void *ptr) { umf_result_t disjoint_pool_get_last_allocation_error(void *pool) { (void)pool; - return TLS_last_allocation_error; } // Define destructor for use with unique_ptr void disjoint_pool_finalize(void *pool) { - disjoint_pool_t *hPool = (disjoint_pool_t *)pool; if (hPool->params.pool_trace > 1) { @@ -937,7 +940,7 @@ void umfDisjointPoolSharedLimitsDestroy( umf_result_t umfDisjointPoolParamsCreate(umf_disjoint_pool_params_handle_t *hParams) { - static const char *DEFAULT_NAME = "disjoint_pool"; + static char *DEFAULT_NAME = "disjoint_pool"; if (!hParams) { LOG_ERR("disjoint pool params handle is NULL"); @@ -951,20 +954,16 @@ umfDisjointPoolParamsCreate(umf_disjoint_pool_params_handle_t *hParams) { return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - params->slab_min_size = 0; - params->max_poolable_size = 0; - params->capacity = 0; - params->min_bucket_size = UMF_DISJOINT_POOL_MIN_BUCKET_DEFAULT_SIZE; - params->cur_pool_size = 0; - params->pool_trace = 0; - params->shared_limits = NULL; - params->name = NULL; - - umf_result_t ret = umfDisjointPoolParamsSetName(params, DEFAULT_NAME); - if (ret != UMF_RESULT_SUCCESS) { - umf_ba_global_free(params); - return ret; - } + *params = (umf_disjoint_pool_params_t){ + .slab_min_size = 0, + .max_poolable_size = 0, + .capacity = 0, + .min_bucket_size = UMF_DISJOINT_POOL_MIN_BUCKET_DEFAULT_SIZE, + .cur_pool_size = 0, + .pool_trace = 0, + .shared_limits = NULL, + .name = {*DEFAULT_NAME}, + }; *hParams = params; @@ -975,7 +974,6 @@ umf_result_t umfDisjointPoolParamsDestroy(umf_disjoint_pool_params_handle_t hParams) { // NOTE: dereferencing hParams when BA is already destroyed leads to crash if (hParams && !umf_ba_global_is_destroyed()) { - umf_ba_global_free(hParams->name); umf_ba_global_free(hParams); } @@ -1067,15 +1065,6 @@ umfDisjointPoolParamsSetName(umf_disjoint_pool_params_handle_t hParams, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - char *newName = umf_ba_global_alloc(sizeof(*newName) * (strlen(name) + 1)); - if (newName == NULL) { - LOG_ERR("cannot allocate memory for disjoint pool name"); - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - umf_ba_global_free(hParams->name); - hParams->name = newName; - strcpy(hParams->name, name); - + strncpy(hParams->name, name, sizeof(hParams->name) - 1); return UMF_RESULT_SUCCESS; } diff --git a/src/pool/pool_disjoint_internal.h b/src/pool/pool_disjoint_internal.h index 2b5de64bc3..43cf73e13d 100644 --- a/src/pool/pool_disjoint_internal.h +++ b/src/pool/pool_disjoint_internal.h @@ -131,7 +131,7 @@ typedef struct umf_disjoint_pool_params_t { umf_disjoint_pool_shared_limits_handle_t shared_limits; // Name used in traces - char *name; + char name[64]; } umf_disjoint_pool_params_t; typedef struct disjoint_pool_t { From 8cf9196f14dfe78a3d258d2e7e199a9de7037c33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Tue, 11 Mar 2025 10:54:54 +0100 Subject: [PATCH 31/47] [CI] Use venv for pip installation in bench Moved on to new Ubuntu and it doesn't support global pip installation. --- .github/workflows/reusable_benchmarks.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable_benchmarks.yml b/.github/workflows/reusable_benchmarks.yml index 15e6b15f4a..69e1f5c601 100644 --- a/.github/workflows/reusable_benchmarks.yml +++ b/.github/workflows/reusable_benchmarks.yml @@ -111,7 +111,9 @@ jobs: - name: Install benchmarking scripts deps run: | - pip install --force-reinstall -r ${{github.workspace}}/sycl-repo/unified-runtime/third_party/benchmark_requirements.txt + python -m venv .venv + source .venv/bin/activate + pip install -r ${{github.workspace}}/sycl-repo/unified-runtime/third_party/benchmark_requirements.txt - name: Set core range and GPU mask run: | @@ -135,6 +137,7 @@ jobs: id: benchmarks working-directory: ${{env.BUILD_DIR}} run: > + source ${{github.workspace}}/.venv/bin/activate && taskset -c ${{ env.CORES }} ${{ github.workspace }}/sycl-repo/unified-runtime/scripts/benchmarks/main.py ~/bench_workdir_umf --umf ${{env.BUILD_DIR}} From eec4f1356fce83c7ba480936768fe0c39c60c4fd Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Wed, 12 Mar 2025 10:01:59 +0000 Subject: [PATCH 32/47] fix for potential out-of-bounds read in tracker --- src/provider/provider_tracking.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index 92d3dd59c6..c5a4b5f1f4 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -1236,9 +1236,9 @@ umf_memory_tracker_handle_t umfMemoryTrackerCreate(void) { err_destroy_ipc_info_allocator: umf_ba_destroy(handle->ipc_info_allocator); err_destroy_alloc_segments_map: - for (int j = i; j >= 0; j--) { - if (handle->alloc_segments_map[j]) { - critnib_delete(handle->alloc_segments_map[j]); + for (i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { + if (handle->alloc_segments_map[i]) { + critnib_delete(handle->alloc_segments_map[i]); } } utils_mutex_destroy_not_free(&handle->splitMergeMutex); From f22aff25e96c3c233b7f4db81fa58f122466ef2d Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Wed, 12 Mar 2025 11:33:10 +0100 Subject: [PATCH 33/47] Log pool pointer in umfFree() --- src/memory_pool.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/memory_pool.c b/src/memory_pool.c index ef2c0fa66b..eb00545228 100644 --- a/src/memory_pool.c +++ b/src/memory_pool.c @@ -111,6 +111,8 @@ void umfPoolDestroy(umf_memory_pool_handle_t hPool) { umf_result_t umfFree(void *ptr) { umf_memory_pool_handle_t hPool = umfPoolByPtr(ptr); if (hPool) { + LOG_DEBUG("calling umfPoolFree(pool=%p, ptr=%p) ...", (void *)hPool, + ptr); return umfPoolFree(hPool, ptr); } return UMF_RESULT_SUCCESS; From ce6c5da7a0be4af53367e40a961e7d8ab30e110a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Wed, 26 Feb 2025 16:29:17 +0100 Subject: [PATCH 34/47] [CI] Enable SLES in multi numa workflow Co-developed-by: opensource-krzysztof --- .github/workflows/reusable_multi_numa.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/reusable_multi_numa.yml b/.github/workflows/reusable_multi_numa.yml index 3c60bebc38..8b342106ef 100644 --- a/.github/workflows/reusable_multi_numa.yml +++ b/.github/workflows/reusable_multi_numa.yml @@ -1,4 +1,4 @@ -# Runs tests on multi-numa machine +# Runs tests on multi-numa machines name: MultiNuma on: [workflow_call] @@ -20,7 +20,7 @@ jobs: strategy: matrix: - os: [ubuntu-22.04, rhel-9.1] + os: [ubuntu-22.04, rhel-9.1, sles-15] build_type: [Debug, Release] shared_library: ['ON', 'OFF'] runs-on: ["DSS-MULTI-NUMA", "DSS-${{matrix.os}}"] @@ -53,16 +53,16 @@ jobs: run: cmake --build ${{github.workspace}}/build -j $(nproc) - name: Run tests - if: matrix.os != 'rhel-9.1' + if: (matrix.os != 'rhel-9.1') && (matrix.os != 'sles-15') working-directory: ${{github.workspace}}/build run: ctest --output-on-failure --test-dir test - # On RHEL, hwloc version is just a little too low. + # On RHEL/SLES, hwloc version is just a little too low. # Skip some tests until we upgrade hwloc and update CMake to properly handle local hwloc installation. # TODO: fix issue #560 # TODO: add issue for -E test_init_teardown - it is not clear why it fails - - name: Run tests (on RHEL) - if: matrix.os == 'rhel-9.1' + - name: Run tests (on RHEL/SLES) + if: (matrix.os == 'rhel-9.1') || (matrix.os == 'sles-15') working-directory: ${{github.workspace}}/build run: | ctest --output-on-failure --test-dir test -E "test_provider_os_memory_multiple_numa_nodes|test_init_teardown" @@ -70,7 +70,7 @@ jobs: --gtest_filter="-*checkModeLocal/*:*checkModePreferredEmptyNodeset/*:testNuma.checkModeInterleave" - name: Run NUMA tests under valgrind - if: matrix.os != 'rhel-9.1' + if: (matrix.os != 'rhel-9.1') && (matrix.os != 'sles-15') run: | ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{env.BUILD_DIR}} memcheck "${{env.NUMA_TESTS}}" ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{env.BUILD_DIR}} drd "${{env.NUMA_TESTS}}" From a234364ab91e40c3784f2e8711dcc3a18d725e2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Wed, 12 Mar 2025 12:33:25 +0100 Subject: [PATCH 35/47] [CI] Update get_system_info.sh zypper part and move the step gathering info to the end (in multi numa CI). --- .github/scripts/get_system_info.sh | 2 +- .github/workflows/reusable_multi_numa.yml | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/scripts/get_system_info.sh b/.github/scripts/get_system_info.sh index be900e2a73..81c54ce980 100755 --- a/.github/scripts/get_system_info.sh +++ b/.github/scripts/get_system_info.sh @@ -15,7 +15,7 @@ function check_L0_version { fi if command -v zypper &> /dev/null; then - zypper se level-zero && return + zypper -n se level-zero || true fi echo "level-zero not installed" diff --git a/.github/workflows/reusable_multi_numa.yml b/.github/workflows/reusable_multi_numa.yml index 8b342106ef..47a48adb20 100644 --- a/.github/workflows/reusable_multi_numa.yml +++ b/.github/workflows/reusable_multi_numa.yml @@ -31,9 +31,6 @@ jobs: with: fetch-depth: 0 - - name: Get information about platform - run: .github/scripts/get_system_info.sh - - name: Configure build run: > cmake @@ -91,3 +88,7 @@ jobs: with: name: ${{env.COVERAGE_NAME}}-${{matrix.os}}-shared-${{matrix.shared_library}} path: ${{env.COVERAGE_DIR}} + + - name: Get information about platform + if: always() + run: .github/scripts/get_system_info.sh From 4cba219fcecd31f1ca02bc9988c9a059cbff6398 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Wed, 12 Mar 2025 14:03:13 +0100 Subject: [PATCH 36/47] Handle CUDA_ERROR_DEINITIALIZED error in cu2umf_result() Signed-off-by: Lukasz Dorau --- src/provider/provider_cuda.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/provider/provider_cuda.c b/src/provider/provider_cuda.c index bb4b3cf644..7d40c534a9 100644 --- a/src/provider/provider_cuda.c +++ b/src/provider/provider_cuda.c @@ -178,6 +178,9 @@ static umf_result_t cu2umf_result(CUresult result) { case CUDA_ERROR_INVALID_VALUE: case CUDA_ERROR_INVALID_HANDLE: return UMF_RESULT_ERROR_INVALID_ARGUMENT; + case CUDA_ERROR_DEINITIALIZED: + LOG_ERR("CUDA driver has been deinitialized"); + return UMF_RESULT_ERROR_OUT_OF_RESOURCES; default: cu_store_last_native_error(result); return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; From 3a08d6c76516d8e37c10bf15b562f558553c46ca Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Wed, 12 Mar 2025 09:33:21 +0000 Subject: [PATCH 37/47] use current ctx and dev by default in CUDA prov --- include/umf/providers/provider_cuda.h | 3 +- src/provider/provider_cuda.c | 40 +++++++++++++++++++++---- test/providers/cuda_helpers.cpp | 12 ++++++++ test/providers/cuda_helpers.h | 2 ++ test/providers/provider_cuda.cpp | 42 ++++++++++++++++++++++++++- 5 files changed, 92 insertions(+), 7 deletions(-) diff --git a/include/umf/providers/provider_cuda.h b/include/umf/providers/provider_cuda.h index e3b81858bb..95f2634fbc 100644 --- a/include/umf/providers/provider_cuda.h +++ b/include/umf/providers/provider_cuda.h @@ -20,7 +20,8 @@ typedef struct umf_cuda_memory_provider_params_t *umf_cuda_memory_provider_params_handle_t; /// @brief Create a struct to store parameters of the CUDA Memory Provider. -/// @param hParams [out] handle to the newly created parameters struct. +/// @param hParams [out] handle to the newly created parameters structure, +/// initialized with the default (current) context and device ID. /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. umf_result_t umfCUDAMemoryProviderParamsCreate( umf_cuda_memory_provider_params_handle_t *hParams); diff --git a/src/provider/provider_cuda.c b/src/provider/provider_cuda.c index bb4b3cf644..a7179defd3 100644 --- a/src/provider/provider_cuda.c +++ b/src/provider/provider_cuda.c @@ -139,6 +139,7 @@ typedef struct cu_ops_t { CUresult (*cuGetErrorName)(CUresult error, const char **pStr); CUresult (*cuGetErrorString)(CUresult error, const char **pStr); CUresult (*cuCtxGetCurrent)(CUcontext *pctx); + CUresult (*cuCtxGetDevice)(CUdevice *device); CUresult (*cuCtxSetCurrent)(CUcontext ctx); CUresult (*cuIpcGetMemHandle)(CUipcMemHandle *pHandle, CUdeviceptr dptr); CUresult (*cuIpcOpenMemHandle)(CUdeviceptr *pdptr, CUipcMemHandle handle, @@ -221,6 +222,8 @@ static void init_cu_global_state(void) { utils_get_symbol_addr(lib_handle, "cuGetErrorString", lib_name); *(void **)&g_cu_ops.cuCtxGetCurrent = utils_get_symbol_addr(lib_handle, "cuCtxGetCurrent", lib_name); + *(void **)&g_cu_ops.cuCtxGetDevice = + utils_get_symbol_addr(lib_handle, "cuCtxGetDevice", lib_name); *(void **)&g_cu_ops.cuCtxSetCurrent = utils_get_symbol_addr(lib_handle, "cuCtxSetCurrent", lib_name); *(void **)&g_cu_ops.cuIpcGetMemHandle = @@ -234,9 +237,9 @@ static void init_cu_global_state(void) { !g_cu_ops.cuMemHostAlloc || !g_cu_ops.cuMemAllocManaged || !g_cu_ops.cuMemFree || !g_cu_ops.cuMemFreeHost || !g_cu_ops.cuGetErrorName || !g_cu_ops.cuGetErrorString || - !g_cu_ops.cuCtxGetCurrent || !g_cu_ops.cuCtxSetCurrent || - !g_cu_ops.cuIpcGetMemHandle || !g_cu_ops.cuIpcOpenMemHandle || - !g_cu_ops.cuIpcCloseMemHandle) { + !g_cu_ops.cuCtxGetCurrent || !g_cu_ops.cuCtxGetDevice || + !g_cu_ops.cuCtxSetCurrent || !g_cu_ops.cuIpcGetMemHandle || + !g_cu_ops.cuIpcOpenMemHandle || !g_cu_ops.cuIpcCloseMemHandle) { LOG_FATAL("Required CUDA symbols not found."); Init_cu_global_state_failed = true; utils_close_library(lib_handle); @@ -260,8 +263,29 @@ umf_result_t umfCUDAMemoryProviderParamsCreate( return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - params_data->cuda_context_handle = NULL; - params_data->cuda_device_handle = -1; + utils_init_once(&cu_is_initialized, init_cu_global_state); + if (Init_cu_global_state_failed) { + LOG_FATAL("Loading CUDA symbols failed"); + return UMF_RESULT_ERROR_DEPENDENCY_UNAVAILABLE; + } + + // initialize context and device to the current ones + CUcontext current_ctx = NULL; + CUresult cu_result = g_cu_ops.cuCtxGetCurrent(¤t_ctx); + if (cu_result == CUDA_SUCCESS) { + params_data->cuda_context_handle = current_ctx; + } else { + params_data->cuda_context_handle = NULL; + } + + CUdevice current_device = -1; + cu_result = g_cu_ops.cuCtxGetDevice(¤t_device); + if (cu_result == CUDA_SUCCESS) { + params_data->cuda_device_handle = current_device; + } else { + params_data->cuda_device_handle = -1; + } + params_data->memory_type = UMF_MEMORY_TYPE_UNKNOWN; params_data->alloc_flags = 0; @@ -342,6 +366,12 @@ static umf_result_t cu_memory_provider_initialize(void *params, } if (cu_params->cuda_context_handle == NULL) { + LOG_ERR("Invalid context handle"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (cu_params->cuda_device_handle < 0) { + LOG_ERR("Invalid device handle"); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } diff --git a/test/providers/cuda_helpers.cpp b/test/providers/cuda_helpers.cpp index a607d7ecb3..3e81c184ff 100644 --- a/test/providers/cuda_helpers.cpp +++ b/test/providers/cuda_helpers.cpp @@ -412,6 +412,18 @@ CUcontext get_mem_context(void *ptr) { return context; } +int get_mem_device(void *ptr) { + int device; + CUresult res = libcu_ops.cuPointerGetAttribute( + &device, CU_POINTER_ATTRIBUTE_DEVICE_ORDINAL, (CUdeviceptr)ptr); + if (res != CUDA_SUCCESS) { + fprintf(stderr, "cuPointerGetAttribute() failed!\n"); + return -1; + } + + return device; +} + CUcontext get_current_context() { CUcontext context; CUresult res = libcu_ops.cuCtxGetCurrent(&context); diff --git a/test/providers/cuda_helpers.h b/test/providers/cuda_helpers.h index e7deb9064c..944e6dbef8 100644 --- a/test/providers/cuda_helpers.h +++ b/test/providers/cuda_helpers.h @@ -48,6 +48,8 @@ unsigned int get_mem_host_alloc_flags(void *ptr); CUcontext get_mem_context(void *ptr); +int get_mem_device(void *ptr); + CUcontext get_current_context(); #ifdef __cplusplus diff --git a/test/providers/provider_cuda.cpp b/test/providers/provider_cuda.cpp index 9c7f76dd10..a7e5dbe5a0 100644 --- a/test/providers/provider_cuda.cpp +++ b/test/providers/provider_cuda.cpp @@ -142,14 +142,15 @@ struct umfCUDAProviderTest memAccessor = nullptr; expected_context = cudaTestHelper.get_test_context(); + expected_device = cudaTestHelper.get_test_device(); params = create_cuda_prov_params(cudaTestHelper.get_test_context(), cudaTestHelper.get_test_device(), memory_type, 0 /* alloc flags */); ASSERT_NE(expected_context, nullptr); + ASSERT_GE(expected_device, 0); switch (memory_type) { case UMF_MEMORY_TYPE_DEVICE: - memAccessor = std::make_unique( cudaTestHelper.get_test_context(), cudaTestHelper.get_test_device()); @@ -178,6 +179,7 @@ struct umfCUDAProviderTest std::unique_ptr memAccessor = nullptr; CUcontext expected_context = nullptr; + int expected_device = -1; umf_usm_memory_type_t expected_memory_type; }; @@ -328,6 +330,44 @@ TEST_P(umfCUDAProviderTest, getPageSizeInvalidArgs) { umfMemoryProviderDestroy(provider); } +TEST_P(umfCUDAProviderTest, cudaProviderDefaultParams) { + umf_cuda_memory_provider_params_handle_t defaultParams = nullptr; + umf_result_t umf_result = umfCUDAMemoryProviderParamsCreate(&defaultParams); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_result = umfCUDAMemoryProviderParamsSetMemoryType(defaultParams, + expected_memory_type); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + // NOTE: we intentionally do not set any context and device params + + umf_memory_provider_handle_t provider = nullptr; + umf_result = umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), + defaultParams, &provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider, nullptr); + + // do single alloc and check if the context and device id of allocated + // memory are correct + + void *ptr = nullptr; + umf_result = umfMemoryProviderAlloc(provider, 128, 0, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + + CUcontext actual_mem_context = get_mem_context(ptr); + ASSERT_EQ(actual_mem_context, expected_context); + + int actual_device = get_mem_device(ptr); + ASSERT_EQ(actual_device, expected_device); + + umf_result = umfMemoryProviderFree(provider, ptr, 128); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfMemoryProviderDestroy(provider); + umfCUDAMemoryProviderParamsDestroy(defaultParams); +} + TEST_P(umfCUDAProviderTest, cudaProviderNullParams) { umf_result_t res = umfCUDAMemoryProviderParamsCreate(nullptr); EXPECT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); From 7173cc5e90d73d12cb731adc2ec5ea3383105d1a Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Wed, 12 Mar 2025 13:47:46 +0100 Subject: [PATCH 38/47] Fix segfault in cu_memory_provider_get_last_native_error() Fix segfault in cu_memory_provider_get_last_native_error() when it is called after a CUDA device was destroyed. Signed-off-by: Lukasz Dorau --- src/provider/provider_cuda.c | 39 +++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/src/provider/provider_cuda.c b/src/provider/provider_cuda.c index 7d40c534a9..c7e2a8fd22 100644 --- a/src/provider/provider_cuda.c +++ b/src/provider/provider_cuda.c @@ -542,22 +542,41 @@ static void cu_memory_provider_get_last_native_error(void *provider, return; } - const char *error_name = 0; - const char *error_string = 0; - g_cu_ops.cuGetErrorName(TLS_last_native_error.native_error, &error_name); - g_cu_ops.cuGetErrorString(TLS_last_native_error.native_error, - &error_string); - + CUresult result; size_t buf_size = 0; - strncpy(TLS_last_native_error.msg_buff, error_name, TLS_MSG_BUF_LEN - 1); - buf_size = strlen(TLS_last_native_error.msg_buff); + const char *error_name = NULL; + const char *error_string = NULL; + + // If the error code is not recognized, + // CUDA_ERROR_INVALID_VALUE will be returned + // and error_name will be set to the NULL address. + result = g_cu_ops.cuGetErrorName(TLS_last_native_error.native_error, + &error_name); + if (result == CUDA_SUCCESS && error_name != NULL) { + strncpy(TLS_last_native_error.msg_buff, error_name, + TLS_MSG_BUF_LEN - 1); + } else { + strncpy(TLS_last_native_error.msg_buff, "cuGetErrorName() failed", + TLS_MSG_BUF_LEN - 1); + } + buf_size = strlen(TLS_last_native_error.msg_buff); strncat(TLS_last_native_error.msg_buff, " - ", TLS_MSG_BUF_LEN - buf_size - 1); buf_size = strlen(TLS_last_native_error.msg_buff); - strncat(TLS_last_native_error.msg_buff, error_string, - TLS_MSG_BUF_LEN - buf_size - 1); + // If the error code is not recognized, + // CUDA_ERROR_INVALID_VALUE will be returned + // and error_string will be set to the NULL address. + result = g_cu_ops.cuGetErrorString(TLS_last_native_error.native_error, + &error_string); + if (result == CUDA_SUCCESS && error_string != NULL) { + strncat(TLS_last_native_error.msg_buff, error_string, + TLS_MSG_BUF_LEN - buf_size - 1); + } else { + strncat(TLS_last_native_error.msg_buff, "cuGetErrorString() failed", + TLS_MSG_BUF_LEN - buf_size - 1); + } *pError = TLS_last_native_error.native_error; *ppMessage = TLS_last_native_error.msg_buff; From 27dc807196da49c98f953a5529e3aca82087b7db Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Thu, 13 Mar 2025 12:20:11 +0100 Subject: [PATCH 39/47] Move C++ helper header to utils --- .github/workflows/reusable_gpu.yml | 4 ++-- scripts/qemu/configs/default.xml | 4 ---- test/c_api/test_ut_asserts.h | 4 ++-- test/common/pool.hpp | 2 +- test/common/provider.hpp | 4 ++-- test/provider_devdax_memory.cpp | 4 ++-- test/provider_file_memory.cpp | 4 ++-- test/provider_fixed_memory.cpp | 2 +- test/provider_os_memory.cpp | 2 +- test/provider_tracking.cpp | 2 +- test/provider_tracking_fixture_tests.cpp | 2 +- {src => test/utils}/cpp_helpers.hpp | 0 12 files changed, 15 insertions(+), 19 deletions(-) rename {src => test/utils}/cpp_helpers.hpp (100%) diff --git a/.github/workflows/reusable_gpu.yml b/.github/workflows/reusable_gpu.yml index a67be2f522..721d85206b 100644 --- a/.github/workflows/reusable_gpu.yml +++ b/.github/workflows/reusable_gpu.yml @@ -99,8 +99,8 @@ jobs: -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF -DUMF_BUILD_${{inputs.name}}_PROVIDER=ON -DUMF_TESTS_FAIL_ON_SKIP=ON - ${{ matrix.os == 'Ubuntu' && matrix.build_type == 'Debug' && '-DUMF_USE_COVERAGE=ON' || '' }} - ${{ matrix.os == 'Windows' && '-DCMAKE_SUPPRESS_REGENERATION=ON' || '' }} + ${{ matrix.os == 'Ubuntu' && matrix.build_type == 'Debug' && '-DUMF_USE_COVERAGE=ON' || '' }} + ${{ matrix.os == 'Windows' && '-DCMAKE_SUPPRESS_REGENERATION=ON' || '' }} - name: Build UMF run: cmake --build ${{env.BUILD_DIR}} --config ${{matrix.build_type}} -j ${{matrix.number_of_processors}} diff --git a/scripts/qemu/configs/default.xml b/scripts/qemu/configs/default.xml index 5654687949..5d3198f60a 100644 --- a/scripts/qemu/configs/default.xml +++ b/scripts/qemu/configs/default.xml @@ -50,16 +50,12 @@ Cell 2 | 0 | 1200MiB | 17, 28, 10 | - - - - diff --git a/test/c_api/test_ut_asserts.h b/test/c_api/test_ut_asserts.h index 834d39bda8..b73f0cd19e 100644 --- a/test/c_api/test_ut_asserts.h +++ b/test/c_api/test_ut_asserts.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023-2024 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -9,7 +9,7 @@ /* The project uses GTEST framework for testing, which is not supported in C - These asserts should NOT be used in other purposes than for testing C API + These asserts should NOT be used in other purposes than for testing C API */ #ifndef UMF_TEST_UT_ASSERTS_H diff --git a/test/common/pool.hpp b/test/common/pool.hpp index a5b4afc15b..165f9b8365 100644 --- a/test/common/pool.hpp +++ b/test/common/pool.hpp @@ -22,8 +22,8 @@ #include #include "base.hpp" -#include "cpp_helpers.hpp" #include "provider.hpp" +#include "utils/cpp_helpers.hpp" namespace umf_test { diff --git a/test/common/provider.hpp b/test/common/provider.hpp index 148f34dc89..f40c0bf640 100644 --- a/test/common/provider.hpp +++ b/test/common/provider.hpp @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023-2024 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -15,8 +15,8 @@ #include "base.hpp" #include "base_alloc_global.h" -#include "cpp_helpers.hpp" #include "test_helpers.h" +#include "utils/cpp_helpers.hpp" namespace umf_test { diff --git a/test/provider_devdax_memory.cpp b/test/provider_devdax_memory.cpp index 7765dd08da..f0f5c21fd9 100644 --- a/test/provider_devdax_memory.cpp +++ b/test/provider_devdax_memory.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2024 Intel Corporation +// Copyright (C) 2024-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -11,8 +11,8 @@ #include "base.hpp" -#include "cpp_helpers.hpp" #include "test_helpers.h" +#include "utils/cpp_helpers.hpp" #include #include diff --git a/test/provider_file_memory.cpp b/test/provider_file_memory.cpp index cfa37be31a..896b95380a 100644 --- a/test/provider_file_memory.cpp +++ b/test/provider_file_memory.cpp @@ -1,11 +1,11 @@ -// Copyright (C) 2024 Intel Corporation +// Copyright (C) 2024-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "base.hpp" -#include "cpp_helpers.hpp" #include "test_helpers.h" +#include "utils/cpp_helpers.hpp" #ifndef _WIN32 #include "test_helpers_linux.h" #endif diff --git a/test/provider_fixed_memory.cpp b/test/provider_fixed_memory.cpp index 1760ca4f74..1d75056ea1 100644 --- a/test/provider_fixed_memory.cpp +++ b/test/provider_fixed_memory.cpp @@ -4,8 +4,8 @@ #include "base.hpp" -#include "cpp_helpers.hpp" #include "test_helpers.h" +#include "utils/cpp_helpers.hpp" #ifndef _WIN32 #include "test_helpers_linux.h" #endif diff --git a/test/provider_os_memory.cpp b/test/provider_os_memory.cpp index 5b647b642c..7fd781208f 100644 --- a/test/provider_os_memory.cpp +++ b/test/provider_os_memory.cpp @@ -4,9 +4,9 @@ #include "base.hpp" -#include "cpp_helpers.hpp" #include "ipcFixtures.hpp" #include "test_helpers.h" +#include "utils/cpp_helpers.hpp" #include #include diff --git a/test/provider_tracking.cpp b/test/provider_tracking.cpp index 864c155649..d289a97965 100644 --- a/test/provider_tracking.cpp +++ b/test/provider_tracking.cpp @@ -4,8 +4,8 @@ #include "base.hpp" -#include "cpp_helpers.hpp" #include "test_helpers.h" +#include "utils/cpp_helpers.hpp" #ifndef _WIN32 #include "test_helpers_linux.h" #endif diff --git a/test/provider_tracking_fixture_tests.cpp b/test/provider_tracking_fixture_tests.cpp index 05b87f87fb..94227757be 100644 --- a/test/provider_tracking_fixture_tests.cpp +++ b/test/provider_tracking_fixture_tests.cpp @@ -9,8 +9,8 @@ #include "base.hpp" #include "provider.hpp" -#include "cpp_helpers.hpp" #include "test_helpers.h" +#include "utils/cpp_helpers.hpp" #ifndef _WIN32 #include "test_helpers_linux.h" #endif diff --git a/src/cpp_helpers.hpp b/test/utils/cpp_helpers.hpp similarity index 100% rename from src/cpp_helpers.hpp rename to test/utils/cpp_helpers.hpp From 52d1f59ba2d8751a0d5961189bd675494e283623 Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Thu, 13 Mar 2025 14:46:25 +0100 Subject: [PATCH 40/47] [tests] Rename umf to umf_test --- test/coarse_lib.cpp | 2 +- test/common/pool.hpp | 4 ++-- test/common/provider.hpp | 9 +++---- test/disjoint_pool_file_prov.cpp | 2 +- test/ipcAPI.cpp | 2 +- test/ipcFixtures.hpp | 30 ++++++++++++------------ test/memoryPoolAPI.cpp | 11 +++++---- test/memoryProviderAPI.cpp | 4 ++-- test/poolFixtures.hpp | 10 ++++---- test/pools/disjoint_pool.cpp | 6 ++--- test/pools/pool_base_alloc.cpp | 11 +++++---- test/pools/scalable_pool.cpp | 6 ++--- test/provider_devdax_memory.cpp | 8 +++---- test/provider_file_memory.cpp | 8 +++---- test/provider_fixed_memory.cpp | 8 +++---- test/provider_os_memory.cpp | 8 +++---- test/provider_tracking.cpp | 12 +++++----- test/provider_tracking_fixture_tests.cpp | 2 +- test/utils/cpp_helpers.hpp | 10 ++++---- 19 files changed, 78 insertions(+), 75 deletions(-) diff --git a/test/coarse_lib.cpp b/test/coarse_lib.cpp index c2e1f9c856..7611833899 100644 --- a/test/coarse_lib.cpp +++ b/test/coarse_lib.cpp @@ -99,7 +99,7 @@ static void coarse_params_set_default(coarse_params_t *coarse_params, } umf_memory_provider_ops_t UMF_MALLOC_MEMORY_PROVIDER_OPS = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); struct CoarseWithMemoryStrategyTest : umf_test::test, diff --git a/test/common/pool.hpp b/test/common/pool.hpp index 165f9b8365..558b9d665a 100644 --- a/test/common/pool.hpp +++ b/test/common/pool.hpp @@ -38,7 +38,7 @@ createPoolChecked(umf_memory_pool_ops_t *ops, } auto wrapPoolUnique(umf_memory_pool_handle_t hPool) { - return umf::pool_unique_handle_t(hPool, &umfPoolDestroy); + return umf_test::pool_unique_handle_t(hPool, &umfPoolDestroy); } bool isReallocSupported(umf_memory_pool_handle_t hPool) { @@ -149,7 +149,7 @@ struct malloc_pool : public pool_base_t { }; umf_memory_pool_ops_t MALLOC_POOL_OPS = - umf::poolMakeCOps(); + umf_test::poolMakeCOps(); static constexpr size_t DEFAULT_DISJOINT_SLAB_MIN_SIZE = 4096; static constexpr size_t DEFAULT_DISJOINT_MAX_POOLABLE_SIZE = 4096; diff --git a/test/common/provider.hpp b/test/common/provider.hpp index f40c0bf640..38fe7336ec 100644 --- a/test/common/provider.hpp +++ b/test/common/provider.hpp @@ -29,7 +29,8 @@ createProviderChecked(umf_memory_provider_ops_t *ops, void *params) { } auto wrapProviderUnique(umf_memory_provider_handle_t hProvider) { - return umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); + return umf_test::provider_unique_handle_t(hProvider, + &umfMemoryProviderDestroy); } typedef struct provider_base_t { @@ -97,7 +98,7 @@ typedef struct provider_base_t { } provider_base_t; umf_memory_provider_ops_t BASE_PROVIDER_OPS = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); struct provider_ba_global : public provider_base_t { umf_result_t alloc(size_t size, size_t align, void **ptr) noexcept { @@ -127,7 +128,7 @@ struct provider_ba_global : public provider_base_t { }; umf_memory_provider_ops_t BA_GLOBAL_PROVIDER_OPS = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); struct provider_mock_out_of_mem : public provider_base_t { provider_ba_global helper_prov; @@ -152,7 +153,7 @@ struct provider_mock_out_of_mem : public provider_base_t { }; umf_memory_provider_ops_t MOCK_OUT_OF_MEM_PROVIDER_OPS = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); } // namespace umf_test diff --git a/test/disjoint_pool_file_prov.cpp b/test/disjoint_pool_file_prov.cpp index b874d2a49b..58e15f5714 100644 --- a/test/disjoint_pool_file_prov.cpp +++ b/test/disjoint_pool_file_prov.cpp @@ -20,7 +20,7 @@ using umf_test::test; #define FILE_PATH ((char *)"tmp_file") umf_memory_provider_ops_t UMF_MALLOC_MEMORY_PROVIDER_OPS = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); struct FileWithMemoryStrategyTest : umf_test::test, diff --git a/test/ipcAPI.cpp b/test/ipcAPI.cpp index 4298963088..c0642dd760 100644 --- a/test/ipcAPI.cpp +++ b/test/ipcAPI.cpp @@ -109,7 +109,7 @@ provider_mock_ipc::allocations_mutex_type provider_mock_ipc::alloc_mutex; provider_mock_ipc::allocations_map_type provider_mock_ipc::allocations; static umf_memory_provider_ops_t IPC_MOCK_PROVIDER_OPS = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); HostMemoryAccessor hostMemoryAccessor; diff --git a/test/ipcFixtures.hpp b/test/ipcFixtures.hpp index 1fc57b9002..cf31ff7584 100644 --- a/test/ipcFixtures.hpp +++ b/test/ipcFixtures.hpp @@ -97,7 +97,7 @@ struct umfIpcTest : umf_test::test, void TearDown() override { test::TearDown(); } - umf::pool_unique_handle_t makePool() { + umf_test::pool_unique_handle_t makePool() { // TODO: The function is similar to poolCreateExt function // from memoryPool.hpp umf_memory_provider_handle_t hProvider = NULL; @@ -147,7 +147,7 @@ struct umfIpcTest : umf_test::test, poolParamsDestroy(poolParams); } - return umf::pool_unique_handle_t(hPool, &umfPoolDestroy); + return umf_test::pool_unique_handle_t(hPool, &umfPoolDestroy); } struct stats_type { @@ -179,7 +179,7 @@ struct umfIpcTest : umf_test::test, std::vector ptrs; constexpr size_t ALLOC_SIZE = 100; constexpr size_t NUM_POINTERS = 100; - umf::pool_unique_handle_t pool = makePool(); + umf_test::pool_unique_handle_t pool = makePool(); ASSERT_NE(pool.get(), nullptr); for (size_t i = 0; i < NUM_POINTERS; ++i) { @@ -237,7 +237,7 @@ struct umfIpcTest : umf_test::test, std::vector ptrs; constexpr size_t ALLOC_SIZE = 100; constexpr size_t NUM_POINTERS = 100; - umf::pool_unique_handle_t pool = makePool(); + umf_test::pool_unique_handle_t pool = makePool(); ASSERT_NE(pool.get(), nullptr); for (size_t i = 0; i < NUM_POINTERS; ++i) { @@ -284,7 +284,7 @@ struct umfIpcTest : umf_test::test, std::vector ptrs; constexpr size_t ALLOC_SIZE = 100; constexpr size_t NUM_POINTERS = 100; - umf::pool_unique_handle_t pool = makePool(); + umf_test::pool_unique_handle_t pool = makePool(); ASSERT_NE(pool.get(), nullptr); for (size_t i = 0; i < NUM_POINTERS; ++i) { @@ -363,7 +363,7 @@ struct umfIpcTest : umf_test::test, std::vector ptrs; constexpr size_t ALLOC_SIZE = 100; constexpr size_t NUM_POINTERS = 100; - umf::pool_unique_handle_t pool = makePool(); + umf_test::pool_unique_handle_t pool = makePool(); ASSERT_NE(pool.get(), nullptr); for (size_t i = 0; i < NUM_POINTERS; ++i) { @@ -432,7 +432,7 @@ struct umfIpcTest : umf_test::test, TEST_P(umfIpcTest, GetIPCHandleSize) { size_t size = 0; - umf::pool_unique_handle_t pool = makePool(); + umf_test::pool_unique_handle_t pool = makePool(); ASSERT_NE(pool.get(), nullptr); umf_result_t ret = umfPoolGetIPCHandleSize(pool.get(), &size); @@ -445,7 +445,7 @@ TEST_P(umfIpcTest, GetIPCHandleSizeInvalidArgs) { umf_result_t ret = umfPoolGetIPCHandleSize(nullptr, &size); EXPECT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); - umf::pool_unique_handle_t pool = makePool(); + umf_test::pool_unique_handle_t pool = makePool(); ASSERT_NE(pool.get(), nullptr); ret = umfPoolGetIPCHandleSize(pool.get(), nullptr); @@ -463,7 +463,7 @@ TEST_P(umfIpcTest, GetIPCHandleInvalidArgs) { ret = umfGetIPCHandle(ptr, &ipcHandle, &handleSize); EXPECT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); - umf::pool_unique_handle_t pool = makePool(); + umf_test::pool_unique_handle_t pool = makePool(); ASSERT_NE(pool.get(), nullptr); ptr = umfPoolMalloc(pool.get(), SIZE); @@ -488,7 +488,7 @@ TEST_P(umfIpcTest, CloseIPCHandleInvalidPtr) { TEST_P(umfIpcTest, BasicFlow) { constexpr size_t SIZE = 100; std::vector expected_data(SIZE); - umf::pool_unique_handle_t pool = makePool(); + umf_test::pool_unique_handle_t pool = makePool(); ASSERT_NE(pool.get(), nullptr); int *ptr = (int *)umfPoolMalloc(pool.get(), SIZE * sizeof(int)); @@ -555,7 +555,7 @@ TEST_P(umfIpcTest, BasicFlow) { TEST_P(umfIpcTest, AllocFreeAllocTest) { constexpr size_t SIZE = 64 * 1024; - umf::pool_unique_handle_t pool = makePool(); + umf_test::pool_unique_handle_t pool = makePool(); ASSERT_NE(pool.get(), nullptr); umf_ipc_handler_handle_t ipcHandler = nullptr; @@ -616,9 +616,9 @@ TEST_P(umfIpcTest, AllocFreeAllocTest) { TEST_P(umfIpcTest, openInTwoIpcHandlers) { constexpr size_t SIZE = 100; std::vector expected_data(SIZE); - umf::pool_unique_handle_t pool1 = makePool(); + umf_test::pool_unique_handle_t pool1 = makePool(); ASSERT_NE(pool1.get(), nullptr); - umf::pool_unique_handle_t pool2 = makePool(); + umf_test::pool_unique_handle_t pool2 = makePool(); ASSERT_NE(pool2.get(), nullptr); umf_ipc_handler_handle_t ipcHandler1 = nullptr; umf_ipc_handler_handle_t ipcHandler2 = nullptr; @@ -715,8 +715,8 @@ TEST_P(umfIpcTest, ConcurrentDestroyIpcHandlers) { constexpr size_t NUM_POOLS = 10; void *ptrs[NUM_ALLOCS]; void *openedPtrs[NUM_POOLS][NUM_ALLOCS]; - std::vector consumerPools; - umf::pool_unique_handle_t producerPool = makePool(); + std::vector consumerPools; + umf_test::pool_unique_handle_t producerPool = makePool(); ASSERT_NE(producerPool.get(), nullptr); for (size_t i = 0; i < NUM_POOLS; ++i) { diff --git a/test/memoryPoolAPI.cpp b/test/memoryPoolAPI.cpp index a949b281f9..e8071a2d81 100644 --- a/test/memoryPoolAPI.cpp +++ b/test/memoryPoolAPI.cpp @@ -125,7 +125,7 @@ TEST_P(umfPoolWithCreateFlagsTest, memoryPoolWithCustomProvider) { return UMF_RESULT_SUCCESS; } }; - umf_memory_pool_ops_t pool_ops = umf::poolMakeCOps(); + umf_memory_pool_ops_t pool_ops = umf_test::poolMakeCOps(); umf_memory_pool_handle_t hPool; auto ret = umfPoolCreate(&pool_ops, hProvider, nullptr, flags, &hPool); @@ -187,8 +187,8 @@ struct tagTest : umf_test::test { createPoolChecked(umfProxyPoolOps(), provider.get(), nullptr)); } - umf::provider_unique_handle_t provider; - umf::pool_unique_handle_t pool; + umf_test::provider_unique_handle_t provider; + umf_test::pool_unique_handle_t pool; }; TEST_F(tagTest, SetAndGet) { @@ -370,7 +370,8 @@ TEST_P(poolInitializeTest, errorPropagation) { return *errorToReturn; } }; - umf_memory_pool_ops_t pool_ops = umf::poolMakeCOps(); + umf_memory_pool_ops_t pool_ops = + umf_test::poolMakeCOps(); umf_memory_pool_handle_t hPool; auto ret = umfPoolCreate(&pool_ops, hProvider, (void *)&this->GetParam(), 0, @@ -420,7 +421,7 @@ TEST_F(test, getLastFailedMemoryProvider) { const char *name; }; umf_memory_provider_ops_t provider_ops = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); auto providerUnique1 = wrapProviderUnique( createProviderChecked(&provider_ops, (void *)"provider1")); diff --git a/test/memoryProviderAPI.cpp b/test/memoryProviderAPI.cpp index 2dc7261f06..720f11b413 100644 --- a/test/memoryProviderAPI.cpp +++ b/test/memoryProviderAPI.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023-2024 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // This file contains tests for UMF provider API @@ -335,7 +335,7 @@ TEST_P(providerInitializeTest, errorPropagation) { } }; umf_memory_provider_ops_t provider_ops = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); umf_memory_provider_handle_t hProvider; auto ret = umfMemoryProviderCreate(&provider_ops, (void *)&this->GetParam(), diff --git a/test/poolFixtures.hpp b/test/poolFixtures.hpp index 6b01769f1b..de5a54685a 100644 --- a/test/poolFixtures.hpp +++ b/test/poolFixtures.hpp @@ -32,7 +32,7 @@ using poolCreateExtParams = pfnPoolParamsDestroy, umf_memory_provider_ops_t *, pfnProviderParamsCreate, pfnProviderParamsDestroy>; -umf::pool_unique_handle_t poolCreateExtUnique(poolCreateExtParams params) { +umf_test::pool_unique_handle_t poolCreateExtUnique(poolCreateExtParams params) { auto [pool_ops, poolParamsCreate, poolParamsDestroy, provider_ops, providerParamsCreate, providerParamsDestroy] = params; @@ -73,7 +73,7 @@ umf::pool_unique_handle_t poolCreateExtUnique(poolCreateExtParams params) { providerParamsDestroy(provider_params); } - return umf::pool_unique_handle_t(hPool, &umfPoolDestroy); + return umf_test::pool_unique_handle_t(hPool, &umfPoolDestroy); } struct umfPoolTest : umf_test::test, @@ -86,7 +86,7 @@ struct umfPoolTest : umf_test::test, void TearDown() override { test::TearDown(); } - umf::pool_unique_handle_t pool; + umf_test::pool_unique_handle_t pool; static constexpr int NTHREADS = 5; static constexpr std::array nonAlignedAllocSizes = {5, 7, 23, 55, @@ -106,7 +106,7 @@ struct umfMultiPoolTest : umf_test::test, void TearDown() override { test::TearDown(); } - std::vector pools; + std::vector pools; }; struct umfMemTest @@ -123,7 +123,7 @@ struct umfMemTest void TearDown() override { test::TearDown(); } - umf::pool_unique_handle_t pool; + umf_test::pool_unique_handle_t pool; int expectedRecycledPoolAllocs; }; diff --git a/test/pools/disjoint_pool.cpp b/test/pools/disjoint_pool.cpp index 4eedce981d..9bdef4f131 100644 --- a/test/pools/disjoint_pool.cpp +++ b/test/pools/disjoint_pool.cpp @@ -40,7 +40,7 @@ TEST_F(test, internals) { } }; umf_memory_provider_ops_t provider_ops = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); auto providerUnique = wrapProviderUnique(createProviderChecked(&provider_ops, nullptr)); @@ -151,7 +151,7 @@ TEST_F(test, freeErrorPropagation) { } }; umf_memory_provider_ops_t provider_ops = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); auto providerUnique = wrapProviderUnique(createProviderChecked(&provider_ops, nullptr)); @@ -206,7 +206,7 @@ TEST_F(test, sharedLimits) { } }; umf_memory_provider_ops_t provider_ops = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); static constexpr size_t SlabMinSize = 1024; static constexpr size_t MaxSize = 4 * SlabMinSize; diff --git a/test/pools/pool_base_alloc.cpp b/test/pools/pool_base_alloc.cpp index ca931bcec3..441ab37ec3 100644 --- a/test/pools/pool_base_alloc.cpp +++ b/test/pools/pool_base_alloc.cpp @@ -17,17 +17,17 @@ struct base_alloc_pool : public umf_test::pool_base_t { void *malloc(size_t size) noexcept { return umf_ba_global_alloc(size); } void *calloc(size_t, size_t) noexcept { - umf::getPoolLastStatusRef() = + umf_test::getPoolLastStatusRef() = UMF_RESULT_ERROR_NOT_SUPPORTED; return NULL; } void *realloc(void *, size_t) noexcept { - umf::getPoolLastStatusRef() = + umf_test::getPoolLastStatusRef() = UMF_RESULT_ERROR_NOT_SUPPORTED; return NULL; } void *aligned_malloc(size_t, size_t) noexcept { - umf::getPoolLastStatusRef() = + umf_test::getPoolLastStatusRef() = UMF_RESULT_ERROR_NOT_SUPPORTED; return NULL; } @@ -39,11 +39,12 @@ struct base_alloc_pool : public umf_test::pool_base_t { return UMF_RESULT_SUCCESS; } umf_result_t get_last_allocation_error() { - return umf::getPoolLastStatusRef(); + return umf_test::getPoolLastStatusRef(); } }; -umf_memory_pool_ops_t BA_POOL_OPS = umf::poolMakeCOps(); +umf_memory_pool_ops_t BA_POOL_OPS = + umf_test::poolMakeCOps(); INSTANTIATE_TEST_SUITE_P(baPool, umfPoolTest, ::testing::Values(poolCreateExtParams{ diff --git a/test/pools/scalable_pool.cpp b/test/pools/scalable_pool.cpp index 14cf5f3059..54c0128a40 100644 --- a/test/pools/scalable_pool.cpp +++ b/test/pools/scalable_pool.cpp @@ -61,7 +61,7 @@ struct umfScalablePoolParamsTest }; static constexpr umf_memory_provider_ops_t VALIDATOR_PROVIDER_OPS = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); umfScalablePoolParamsTest() : expected_params{0, false}, params(nullptr) {} void SetUp() override { @@ -82,7 +82,7 @@ struct umfScalablePoolParamsTest test::TearDown(); } - umf::pool_unique_handle_t makePool() { + umf_test::pool_unique_handle_t makePool() { umf_memory_provider_handle_t hProvider = nullptr; umf_memory_pool_handle_t hPool = nullptr; @@ -94,7 +94,7 @@ struct umfScalablePoolParamsTest UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &hPool); EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - return umf::pool_unique_handle_t(hPool, &umfPoolDestroy); + return umf_test::pool_unique_handle_t(hPool, &umfPoolDestroy); } void allocFreeFlow() { diff --git a/test/provider_devdax_memory.cpp b/test/provider_devdax_memory.cpp index f0f5c21fd9..6efeef90cb 100644 --- a/test/provider_devdax_memory.cpp +++ b/test/provider_devdax_memory.cpp @@ -46,7 +46,7 @@ static int compare_native_error_str(const char *message, int error) { using providerCreateExtParams = std::tuple; static void providerCreateExt(providerCreateExtParams params, - umf::provider_unique_handle_t *handle) { + umf_test::provider_unique_handle_t *handle) { umf_memory_provider_handle_t hProvider = nullptr; auto [provider_ops, provider_params] = params; @@ -55,8 +55,8 @@ static void providerCreateExt(providerCreateExtParams params, ASSERT_EQ(ret, UMF_RESULT_SUCCESS); ASSERT_NE(hProvider, nullptr); - *handle = - umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); + *handle = umf_test::provider_unique_handle_t(hProvider, + &umfMemoryProviderDestroy); } struct umfProviderTest @@ -74,7 +74,7 @@ struct umfProviderTest void TearDown() override { test::TearDown(); } - umf::provider_unique_handle_t provider; + umf_test::provider_unique_handle_t provider; size_t page_size; size_t page_plus_64; }; diff --git a/test/provider_file_memory.cpp b/test/provider_file_memory.cpp index 896b95380a..bcc9d26453 100644 --- a/test/provider_file_memory.cpp +++ b/test/provider_file_memory.cpp @@ -42,7 +42,7 @@ static int compare_native_error_str(const char *message, int error) { using providerCreateExtParams = std::tuple; static void providerCreateExt(providerCreateExtParams params, - umf::provider_unique_handle_t *handle) { + umf_test::provider_unique_handle_t *handle) { umf_memory_provider_handle_t hProvider = nullptr; auto [provider_ops, provider_params] = params; @@ -51,8 +51,8 @@ static void providerCreateExt(providerCreateExtParams params, ASSERT_EQ(ret, UMF_RESULT_SUCCESS); ASSERT_NE(hProvider, nullptr); - *handle = - umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); + *handle = umf_test::provider_unique_handle_t(hProvider, + &umfMemoryProviderDestroy); } struct FileProviderParamsDefault @@ -70,7 +70,7 @@ struct FileProviderParamsDefault void TearDown() override { test::TearDown(); } - umf::provider_unique_handle_t provider; + umf_test::provider_unique_handle_t provider; size_t page_size; size_t page_plus_64; }; diff --git a/test/provider_fixed_memory.cpp b/test/provider_fixed_memory.cpp index 1d75056ea1..dac651435a 100644 --- a/test/provider_fixed_memory.cpp +++ b/test/provider_fixed_memory.cpp @@ -41,7 +41,7 @@ static int compare_native_error_str(const char *message, int error) { using providerCreateExtParams = std::tuple; static void providerCreateExt(providerCreateExtParams params, - umf::provider_unique_handle_t *handle) { + umf_test::provider_unique_handle_t *handle) { umf_memory_provider_handle_t hProvider = nullptr; auto [provider_ops, provider_params] = params; @@ -50,8 +50,8 @@ static void providerCreateExt(providerCreateExtParams params, ASSERT_EQ(ret, UMF_RESULT_SUCCESS); ASSERT_NE(hProvider, nullptr); - *handle = - umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); + *handle = umf_test::provider_unique_handle_t(hProvider, + &umfMemoryProviderDestroy); } struct FixedProviderTest @@ -138,7 +138,7 @@ struct FixedProviderTest } } - umf::provider_unique_handle_t provider; + umf_test::provider_unique_handle_t provider; size_t page_size; size_t page_plus_64; void *memory_buffer = nullptr; diff --git a/test/provider_os_memory.cpp b/test/provider_os_memory.cpp index 7fd781208f..f3552b9236 100644 --- a/test/provider_os_memory.cpp +++ b/test/provider_os_memory.cpp @@ -47,7 +47,7 @@ static int compare_native_error_str(const char *message, int error) { using providerCreateExtParams = std::tuple; static void providerCreateExt(providerCreateExtParams params, - umf::provider_unique_handle_t *handle) { + umf_test::provider_unique_handle_t *handle) { umf_memory_provider_handle_t hProvider = nullptr; auto [provider_ops, provider_params] = params; @@ -56,8 +56,8 @@ static void providerCreateExt(providerCreateExtParams params, ASSERT_EQ(ret, UMF_RESULT_SUCCESS); ASSERT_NE(hProvider, nullptr); - *handle = - umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); + *handle = umf_test::provider_unique_handle_t(hProvider, + &umfMemoryProviderDestroy); } struct umfProviderTest @@ -75,7 +75,7 @@ struct umfProviderTest void TearDown() override { test::TearDown(); } - umf::provider_unique_handle_t provider; + umf_test::provider_unique_handle_t provider; size_t page_size; size_t page_plus_64; }; diff --git a/test/provider_tracking.cpp b/test/provider_tracking.cpp index d289a97965..55acc452cf 100644 --- a/test/provider_tracking.cpp +++ b/test/provider_tracking.cpp @@ -22,7 +22,7 @@ using umf_test::test; using providerCreateExtParams = std::tuple; static void providerCreateExt(providerCreateExtParams params, - umf::provider_unique_handle_t *handle) { + umf_test::provider_unique_handle_t *handle) { umf_memory_provider_handle_t hProvider = nullptr; auto [provider_ops, provider_params] = params; @@ -31,8 +31,8 @@ static void providerCreateExt(providerCreateExtParams params, ASSERT_EQ(ret, UMF_RESULT_SUCCESS); ASSERT_NE(hProvider, nullptr); - *handle = - umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); + *handle = umf_test::provider_unique_handle_t(hProvider, + &umfMemoryProviderDestroy); } struct TrackingProviderTest @@ -68,7 +68,7 @@ struct TrackingProviderTest 0, &hPool); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - pool = umf::pool_unique_handle_t(hPool, &umfPoolDestroy); + pool = umf_test::pool_unique_handle_t(hPool, &umfPoolDestroy); } void TearDown() override { @@ -79,8 +79,8 @@ struct TrackingProviderTest test::TearDown(); } - umf::provider_unique_handle_t provider; - umf::pool_unique_handle_t pool; + umf_test::provider_unique_handle_t provider; + umf_test::pool_unique_handle_t pool; size_t page_size; size_t page_plus_64; void *memory_buffer = nullptr; diff --git a/test/provider_tracking_fixture_tests.cpp b/test/provider_tracking_fixture_tests.cpp index 94227757be..d81d4f8b1d 100644 --- a/test/provider_tracking_fixture_tests.cpp +++ b/test/provider_tracking_fixture_tests.cpp @@ -47,7 +47,7 @@ struct provider_from_pool : public umf_test::provider_base_t { }; umf_memory_provider_ops_t PROVIDER_FROM_POOL_OPS = - umf::providerMakeCOps(); + umf_test::providerMakeCOps(); static void *providerFromPoolParamsCreate(void) { umf_file_memory_provider_params_handle_t paramsFile = NULL; diff --git a/test/utils/cpp_helpers.hpp b/test/utils/cpp_helpers.hpp index 85e81c502b..037c633c17 100644 --- a/test/utils/cpp_helpers.hpp +++ b/test/utils/cpp_helpers.hpp @@ -7,8 +7,8 @@ * */ -#ifndef UMF_HELPERS_HPP -#define UMF_HELPERS_HPP 1 +#ifndef UMF_TEST_HELPERS_HPP +#define UMF_TEST_HELPERS_HPP 1 #include #include @@ -22,7 +22,7 @@ #include #include -namespace umf { +namespace umf_test { using pool_unique_handle_t = std::unique_ptr umf_result_t &getPoolLastStatusRef() { return last_status; } -} // namespace umf +} // namespace umf_test -#endif /* UMF_HELPERS_HPP */ +#endif /* UMF_TEST_HELPERS_HPP */ From b8e7246104955ff6839c1860de0c09c19373579f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Plewa?= Date: Thu, 13 Mar 2025 17:12:13 +0100 Subject: [PATCH 41/47] update benchmark scripts --- .github/workflows/reusable_benchmarks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/reusable_benchmarks.yml b/.github/workflows/reusable_benchmarks.yml index 69e1f5c601..d3296beb2c 100644 --- a/.github/workflows/reusable_benchmarks.yml +++ b/.github/workflows/reusable_benchmarks.yml @@ -103,9 +103,9 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: intel/llvm - # add preloaded UMF benchmarks - # https://github.com/intel/llvm/pull/17278 - ref: b2f9dab5266d227cc9eb19af1b54c5bdc50221d1 + # [BENCHMARK] fix default timeout parameter + # https://github.com/intel/llvm/pull/17412 + ref: 357e9e0b253b7eba105d044e38452b3c09169f8a path: sycl-repo fetch-depth: 1 From 8cd338315b8e8295cb4416c5ef64ec7e48070432 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 14 Mar 2025 14:10:52 +0100 Subject: [PATCH 42/47] Enable jemalloc pool test with Fixed provider --- test/pools/jemalloc_pool.cpp | 42 ++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/test/pools/jemalloc_pool.cpp b/test/pools/jemalloc_pool.cpp index e282be3161..8112f36bf4 100644 --- a/test/pools/jemalloc_pool.cpp +++ b/test/pools/jemalloc_pool.cpp @@ -11,9 +11,7 @@ using umf_test::test; using namespace umf_test; -using os_params_unique_handle_t = - std::unique_ptr; +using void_unique_ptr = std::unique_ptr; void *createOsMemoryProviderParams() { umf_os_memory_provider_params_handle_t params = nullptr; @@ -30,11 +28,43 @@ umf_result_t destroyOsMemoryProviderParams(void *params) { (umf_os_memory_provider_params_handle_t)params); } +void *createFixedMemoryProviderParams() { + // Allocate a memory buffer to use with the fixed memory provider. + // The umfPoolTest.malloc_compliance test requires a lot of memory. + size_t memory_size = (1UL << 31); + static void_unique_ptr memory_buffer = + void_unique_ptr(malloc(memory_size), free); + if (memory_buffer.get() == NULL) { + throw std::runtime_error( + "Failed to allocate memory for Fixed memory provider"); + } + + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result_t res = umfFixedMemoryProviderParamsCreate( + ¶ms, memory_buffer.get(), memory_size); + if (res != UMF_RESULT_SUCCESS) { + throw std::runtime_error( + "Failed to create Fixed memory provider params"); + } + + return params; +} + +umf_result_t destroyFixedMemoryProviderParams(void *params) { + return umfFixedMemoryProviderParamsDestroy( + (umf_fixed_memory_provider_params_handle_t)params); +} + INSTANTIATE_TEST_SUITE_P( jemallocPoolTest, umfPoolTest, - ::testing::Values(poolCreateExtParams{ - umfJemallocPoolOps(), nullptr, nullptr, umfOsMemoryProviderOps(), - createOsMemoryProviderParams, destroyOsMemoryProviderParams})); + ::testing::Values(poolCreateExtParams{umfJemallocPoolOps(), nullptr, + nullptr, umfOsMemoryProviderOps(), + createOsMemoryProviderParams, + destroyOsMemoryProviderParams}, + poolCreateExtParams{umfJemallocPoolOps(), nullptr, + nullptr, umfFixedMemoryProviderOps(), + createFixedMemoryProviderParams, + destroyFixedMemoryProviderParams})); // this test makes sure that jemalloc does not use // memory provider to allocate metadata (and hence From fcf2c8d654fb5748d7d3520e862449d258767a66 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 17 Mar 2025 10:04:23 +0100 Subject: [PATCH 43/47] Install libnuma-dev in the proxy lib workflow Install libnuma-dev in the proxy lib workflow, because it is required. It has worked correctly so far most probably, because libnuma-dev has been installed as a dependency of libhwloc-dev. It failed when libhwloc-dev was not installed. Signed-off-by: Lukasz Dorau --- .github/workflows/reusable_proxy_lib.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_proxy_lib.yml b/.github/workflows/reusable_proxy_lib.yml index 363e665264..c519be95b8 100644 --- a/.github/workflows/reusable_proxy_lib.yml +++ b/.github/workflows/reusable_proxy_lib.yml @@ -32,7 +32,7 @@ jobs: - name: Install apt packages run: | sudo apt-get update - sudo apt-get install -y cmake libhwloc-dev libtbb-dev lcov + sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev lcov - name: Configure build run: > From fc68be82643b590396a1c6d53a82a04693bdbc74 Mon Sep 17 00:00:00 2001 From: Sergei Vinogradov Date: Mon, 17 Mar 2025 10:53:45 +0100 Subject: [PATCH 44/47] Suppress false-postive in IPC tests under helgrind --- test/supp/helgrind-test_ipc.supp | 1 + test/supp/helgrind-test_ipc_max_opened_limit.supp | 1 + 2 files changed, 2 insertions(+) diff --git a/test/supp/helgrind-test_ipc.supp b/test/supp/helgrind-test_ipc.supp index 04f3a91993..25ae87ea43 100644 --- a/test/supp/helgrind-test_ipc.supp +++ b/test/supp/helgrind-test_ipc.supp @@ -19,6 +19,7 @@ [false-positive] Double check locking pattern in trackingOpenIpcHandle Helgrind:Race fun:utils_atomic_store_release_ptr + fun:upstreamOpenIPCHandle fun:trackingOpenIpcHandle fun:umfMemoryProviderOpenIPCHandle fun:umfOpenIPCHandle diff --git a/test/supp/helgrind-test_ipc_max_opened_limit.supp b/test/supp/helgrind-test_ipc_max_opened_limit.supp index 04f3a91993..25ae87ea43 100644 --- a/test/supp/helgrind-test_ipc_max_opened_limit.supp +++ b/test/supp/helgrind-test_ipc_max_opened_limit.supp @@ -19,6 +19,7 @@ [false-positive] Double check locking pattern in trackingOpenIpcHandle Helgrind:Race fun:utils_atomic_store_release_ptr + fun:upstreamOpenIPCHandle fun:trackingOpenIpcHandle fun:umfMemoryProviderOpenIPCHandle fun:umfOpenIPCHandle From fb28a16413ff1b1dcd326aebf65300f82130c58a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Plewa?= Date: Mon, 17 Mar 2025 14:53:30 +0100 Subject: [PATCH 45/47] fix jemalloc benchmark with fixedprovider --- benchmark/benchmark.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index 377a38fcf3..ea546422e5 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -171,7 +171,8 @@ UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, jemalloc_pool_fixedprovider, fixed_alloc_size, pool_allocator>); -UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, jemalloc_pool_fix) +UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, + jemalloc_pool_fixedprovider) ->Apply(&default_multiple_alloc_fix_size) ->Apply(&multithreaded); @@ -179,7 +180,8 @@ UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, jemalloc_pool_uniform_fixedprovider, uniform_alloc_size, pool_allocator>); -UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, jemalloc_pool_uniform) +UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, + jemalloc_pool_uniform_fixedprovider) ->Apply(&default_multiple_alloc_uniform_size) ->Apply(&multithreaded); From ac6a2b254a65b39cfbb693fbdeba0cb60716b8ed Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 27 Feb 2025 07:59:42 +0100 Subject: [PATCH 46/47] Remove old SHM files left from the previous runs, because of crashes Remove old SHM files /tmp/umf_file_provider_* (at the beginning of the test) left from the previous runs, because of crashes of the ipc_file_prov test. Signed-off-by: Lukasz Dorau --- test/ipc_file_prov.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/ipc_file_prov.sh b/test/ipc_file_prov.sh index 629b2cbb79..ffb849f25a 100755 --- a/test/ipc_file_prov.sh +++ b/test/ipc_file_prov.sh @@ -9,7 +9,12 @@ set -e -FILE_NAME="/tmp/umf_file_provider_$$" +FILE_BASE="/tmp/umf_file_provider" + +# remove old SHM files (left from the previous runs, because of crashes) +rm -f ${FILE_BASE}* + +FILE_NAME="${FILE_BASE}_$$" # port should be a number from the range <1024, 65535> PORT=$(( 1024 + ( $$ % ( 65535 - 1024 )))) From e316fdccfbd04965b00351456a308a47774050fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Plewa?= Date: Thu, 6 Mar 2025 14:04:29 +0100 Subject: [PATCH 47/47] increase number of threads in benchmarks --- .github/workflows/reusable_benchmarks.yml | 1 + benchmark/benchmark.cpp | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/reusable_benchmarks.yml b/.github/workflows/reusable_benchmarks.yml index a7c9e5e285..4cf5325ab2 100644 --- a/.github/workflows/reusable_benchmarks.yml +++ b/.github/workflows/reusable_benchmarks.yml @@ -137,6 +137,7 @@ jobs: ~/bench_workdir_umf --umf ${{env.BUILD_DIR}} --compare baseline + --timeout 3000 ${{ inputs.upload_report && '--output-html' || '' }} ${{ inputs.pr_no != 0 && '--output-markdown' || '' }} ${{ inputs.bench_script_params }} diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index 60636a559f..fb9cf68f46 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -31,6 +31,8 @@ // Refer to the 'argsName()' function in each component to find detailed descriptions of these arguments. static void multithreaded(benchmark::internal::Benchmark *benchmark) { + benchmark->Threads(12); + benchmark->Threads(8); benchmark->Threads(4); benchmark->Threads(1); }