diff --git a/.github/docker/ubuntu-20.04.Dockerfile b/.github/docker/ubuntu-20.04.Dockerfile index 069deeac93..a6a45a8c1b 100644 --- a/.github/docker/ubuntu-20.04.Dockerfile +++ b/.github/docker/ubuntu-20.04.Dockerfile @@ -24,7 +24,6 @@ ARG BASE_DEPS="\ # UMF's dependencies ARG UMF_DEPS="\ - libjemalloc-dev \ libhwloc-dev \ libtbb-dev" @@ -34,6 +33,7 @@ ARG TEST_DEPS="\ # Miscellaneous for our builds/CI (optional) ARG MISC_DEPS="\ + automake \ clang \ g++-7 \ python3-pip \ diff --git a/.github/docker/ubuntu-22.04.Dockerfile b/.github/docker/ubuntu-22.04.Dockerfile index 08d546083d..75c71c526c 100644 --- a/.github/docker/ubuntu-22.04.Dockerfile +++ b/.github/docker/ubuntu-22.04.Dockerfile @@ -24,7 +24,6 @@ ARG BASE_DEPS="\ # UMF's dependencies ARG UMF_DEPS="\ - libjemalloc-dev \ libhwloc-dev \ libtbb-dev" @@ -34,6 +33,7 @@ ARG TEST_DEPS="\ # Miscellaneous for our builds/CI (optional) ARG MISC_DEPS="\ + automake \ clang \ python3-pip \ sudo \ diff --git a/.github/scripts/run-codespell.py b/.github/scripts/run-codespell.py new file mode 100644 index 0000000000..b87bf37bd5 --- /dev/null +++ b/.github/scripts/run-codespell.py @@ -0,0 +1,40 @@ +""" + Copyright (C) 2024 Intel Corporation + + Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" + +import subprocess # nosec B404 +import logging +import sys + +logging.basicConfig( + level=logging.INFO, format="[%(levelname)s]: [%(asctime)s] %(message)s" +) + + +def codespell_scan(): + try: + codespell_result = subprocess.run( # nosec + [ + "codespell", + "-H", + "--quiet-level=3", + "--skip=./.git,./.venv,./.github/workflows/.spellcheck-conf.toml", + ], + text=True, + stdout=subprocess.PIPE, + ) + if codespell_result.returncode != 0: + for line in codespell_result.stdout.splitlines(): + logging.error(line.strip()) + sys.exit(1) + else: + logging.info("No spelling errors found") + except subprocess.CalledProcessError as ex: + logging.error(ex) + sys.exit(1) + + +codespell_scan() diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index dfa03fc4f4..531a463c77 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -31,7 +31,7 @@ jobs: - name: Install apt packages run: | sudo apt-get update - sudo apt-get install -y cmake hwloc libhwloc-dev libjemalloc-dev libnuma-dev libtbb-dev + sudo apt-get install -y cmake hwloc libhwloc-dev libnuma-dev libtbb-dev - name: Download Coverity run: | diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3d9bfc29b4..165cc1754f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -14,44 +14,14 @@ permissions: contents: read jobs: - build: - name: Build docs - runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }} - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Install doxygen - run: | - sudo apt-get update - sudo apt-get install -y doxygen - - # Latest distros do not allow global pip installation - - name: Install Python requirements in venv - run: | - python3 -m venv .venv - . .venv/bin/activate - echo "$PATH" >> $GITHUB_PATH - python3 -m pip install -r third_party/requirements.txt - - - name: Setup PATH for python - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Build the documentation - working-directory: scripts - run: python3 generate_docs.py - - - name: Upload artifact - uses: actions/upload-pages-artifact@0252fc4ba7626f0298f0cf00902a25c6afc77fa8 # v3.0.0 - with: - path: docs/html + DocsBuild: + uses: ./.github/workflows/reusable_docs_build.yml + with: + upload: true - deploy: + DocsDeploy: name: Deploy docs to GitHub Pages - needs: build + needs: DocsBuild permissions: pages: write diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 281ae00615..46543fac83 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -67,7 +67,7 @@ jobs: - name: Install apt packages run: | sudo apt-get update - sudo apt-get install -y cmake hwloc libhwloc-dev libjemalloc-dev libnuma-dev libtbb-dev valgrind + sudo apt-get install -y cmake hwloc libhwloc-dev libnuma-dev libtbb-dev valgrind - name: Configure CMake run: > diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index ced48e0c72..a106472c36 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -7,8 +7,6 @@ permissions: contents: read env: - # for installation testing - it should match with version set in CMake - UMF_VERSION: 0.10.1 BUILD_DIR : "${{github.workspace}}/build" INSTL_DIR : "${{github.workspace}}/../install-dir" COVERAGE_DIR : "${{github.workspace}}/coverage" @@ -76,15 +74,15 @@ jobs: disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' # test icx compiler - # - os: 'ubuntu-22.04' - # build_type: Release - # compiler: {c: icx, cxx: icpx} - # shared_library: 'ON' - # level_zero_provider: 'ON' - # cuda_provider: 'ON' - # install_tbb: 'ON' - # disable_hwloc: 'OFF' - # link_hwloc_statically: 'OFF' + - os: 'ubuntu-22.04' + build_type: Release + compiler: {c: icx, cxx: icpx} + shared_library: 'ON' + level_zero_provider: 'ON' + cuda_provider: 'ON' + install_tbb: 'ON' + disable_hwloc: 'OFF' + link_hwloc_statically: 'OFF' # test without installing TBB - os: 'ubuntu-22.04' build_type: Release @@ -124,7 +122,7 @@ jobs: - name: Install apt packages run: | sudo apt-get update - sudo apt-get install -y clang cmake libnuma-dev libjemalloc-dev lcov + sudo apt-get install -y clang cmake libnuma-dev lcov - name: Install TBB apt package if: matrix.install_tbb == 'ON' @@ -147,8 +145,10 @@ jobs: - name: Install libhwloc run: .github/scripts/install_hwloc.sh - - name: Set ptrace value for IPC test - run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" + - name: Get UMF version + run: | + VERSION=$(git describe --tags --abbrev=0 | grep -oP '\d+\.\d+\.\d+') + echo "UMF_VERSION=$VERSION" >> $GITHUB_ENV - name: Configure build run: > @@ -179,8 +179,7 @@ jobs: - name: Run tests working-directory: ${{env.BUILD_DIR}} run: | - ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh' || true }} - ctest --output-on-failure # run all tests for better coverage + LD_LIBRARY_PATH=${{env.BUILD_DIR}}/lib/ ctest --output-on-failure # run all tests for better coverage - name: Check coverage if: ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' }} @@ -209,7 +208,6 @@ jobs: --install-dir ${{env.INSTL_DIR}} --build-type ${{matrix.build_type}} --disjoint-pool - --jemalloc-pool ${{ matrix.install_tbb == 'ON' && matrix.disable_hwloc != 'ON' && matrix.shared_library == 'ON' && '--proxy' || '' }} --umf-version ${{env.UMF_VERSION}} ${{ matrix.shared_library == 'ON' && '--shared-library' || '' }} @@ -267,6 +265,12 @@ jobs: run: vcpkg install shell: pwsh # Specifies PowerShell as the shell for running the script. + - name: Get UMF version + run: | + $version = (git describe --tags --abbrev=0 | Select-String -Pattern '\d+\.\d+\.\d+').Matches.Value + echo "UMF_VERSION=$version" >> $env:GITHUB_ENV + shell: pwsh + - name: Configure build run: > cmake @@ -300,7 +304,6 @@ jobs: --install-dir ${{env.INSTL_DIR}} --build-type ${{matrix.build_type}} --disjoint-pool - --jemalloc-pool ${{matrix.shared_library == 'ON' && '--proxy' || '' }} --umf-version ${{env.UMF_VERSION}} ${{ matrix.shared_library == 'ON' && '--shared-library' || ''}} @@ -335,7 +338,7 @@ jobs: -B ${{env.BUILD_DIR}} -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DUMF_BUILD_SHARED_LIBRARY=ON - -DUMF_BUILD_EXAMPLES=OFF + -DUMF_BUILD_EXAMPLES=ON -DUMF_FORMAT_CODE_STYLE=OFF -DUMF_DEVELOPER_MODE=ON -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON @@ -378,7 +381,7 @@ jobs: -B ${{env.BUILD_DIR}} -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DUMF_BUILD_SHARED_LIBRARY=OFF - -DUMF_BUILD_EXAMPLES=OFF + -DUMF_BUILD_EXAMPLES=ON -DUMF_FORMAT_CODE_STYLE=OFF -DUMF_DEVELOPER_MODE=ON -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON @@ -468,8 +471,17 @@ jobs: echo "$PATH" >> $GITHUB_PATH python3 -m pip install -r third_party/requirements.txt + - name: Install dependencies + run: brew install jemalloc tbb automake + - name: Install hwloc - run: brew install hwloc jemalloc tbb + if: matrix.os == 'macos-14' + run: brew install hwloc + + - name: Get UMF version + run: | + VERSION=$(git describe --tags --abbrev=0 | grep -Eo '\d+\.\d+\.\d+') + echo "UMF_VERSION=$VERSION" >> $GITHUB_ENV - name: Configure build run: > @@ -484,6 +496,7 @@ jobs: -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON -DUMF_BUILD_SHARED_LIBRARY=ON -DUMF_TESTS_FAIL_ON_SKIP=ON + ${{ matrix.os != 'macos-14' && '-DUMF_LINK_HWLOC_STATICALLY=ON' || '' }} - name: Build UMF run: cmake --build ${{env.BUILD_DIR}} -j $(sysctl -n hw.logicalcpu) @@ -495,7 +508,6 @@ jobs: --install-dir ${{env.INSTL_DIR}} --build-type ${{env.BUILD_TYPE}} --disjoint-pool - --jemalloc-pool --proxy --umf-version ${{env.UMF_VERSION}} --shared-library diff --git a/.github/workflows/reusable_benchmarks.yml b/.github/workflows/reusable_benchmarks.yml index 41710029c8..ed6a482947 100644 --- a/.github/workflows/reusable_benchmarks.yml +++ b/.github/workflows/reusable_benchmarks.yml @@ -34,7 +34,7 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt-get update - sudo apt-get install -y cmake libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev + sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev - name: Initialize vcpkg if: matrix.os == 'windows-latest' diff --git a/.github/workflows/reusable_checks.yml b/.github/workflows/reusable_checks.yml index e3e264b0db..a7602d2696 100644 --- a/.github/workflows/reusable_checks.yml +++ b/.github/workflows/reusable_checks.yml @@ -21,7 +21,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y black cmake clang-format-15 cmake-format libhwloc-dev + sudo apt-get install -y black cmake clang-format-15 cmake-format libhwloc-dev doxygen # Latest distros do not allow global pip installation - name: Install Python requirements in venv @@ -29,7 +29,8 @@ jobs: python3 -m venv .venv . .venv/bin/activate echo "$PATH" >> $GITHUB_PATH - python3 -m pip install bandit + python3 -m pip install -r third_party/requirements.txt + python3 -m pip install bandit codespell - name: Configure CMake run: > @@ -52,11 +53,24 @@ jobs: - name: Check Python formatting run: cmake --build build --target black-format-check + - name: Run check-license + run: | + ./scripts/check_license/check_headers.sh . "Apache-2.0 WITH LLVM-exception" -v + - name: Run a spell check uses: crate-ci/typos@b63f421581dce830bda2f597a678cb7776b41877 # v1.18.2 with: config: ./.github/workflows/.spellcheck-conf.toml + - name: Run codespell + run: python3 ./.github/scripts/run-codespell.py + + - name: Check spelling in docs + run: | + cmake -B build + cmake --build build --target docs + sphinx-build -b spelling ./build/docs_build/config ./build/docs_build/spelling_log -W + # Run Bandit recursively, but omit _deps directory (with 3rd party code) and python's venv - name: Run Bandit run: python3 -m bandit -r . -x '/_deps/,/.venv/' diff --git a/.github/workflows/reusable_codeql.yml b/.github/workflows/reusable_codeql.yml index e764563103..046c320817 100644 --- a/.github/workflows/reusable_codeql.yml +++ b/.github/workflows/reusable_codeql.yml @@ -62,7 +62,7 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt-get update - sudo apt-get install -y cmake clang libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev + sudo apt-get install -y cmake clang libhwloc-dev libnuma-dev libtbb-dev # Latest distros do not allow global pip installation - name: "[Lin] Install Python requirements in venv" diff --git a/.github/workflows/reusable_dax.yml b/.github/workflows/reusable_dax.yml index f7c5d0d21a..f7f4fbe508 100644 --- a/.github/workflows/reusable_dax.yml +++ b/.github/workflows/reusable_dax.yml @@ -31,6 +31,7 @@ env: INSTL_DIR : "${{github.workspace}}/../install-dir" COVERAGE_DIR : "${{github.workspace}}/coverage" COVERAGE_NAME : "exports-coverage-dax" + DAX_TESTS: "./test/umf_test-provider_file_memory ./test/umf_test-provider_devdax_memory" jobs: dax: @@ -106,8 +107,6 @@ jobs: UMF_TESTS_FSDAX_PATH_2=${{env.UMF_TESTS_FSDAX_PATH_2}} ctest -C ${{matrix.build_type}} -V -R "file|fsdax" - # TODO: enable the provider_devdax_memory_ipc test when the IPC tests with the proxy library are fixed - # see the issue: https://github.com/oneapi-src/unified-memory-framework/issues/864 - name: Run the DEVDAX tests with the proxy library # proxy library is built only if libumf is a shared library if: ${{ matrix.shared_library == 'ON' }} @@ -116,10 +115,8 @@ jobs: LD_PRELOAD=./lib/libumf_proxy.so UMF_TESTS_DEVDAX_PATH="/dev/dax${{env.DEVDAX_NAMESPACE}}" UMF_TESTS_DEVDAX_SIZE="$(ndctl list --namespace=namespace${{env.DEVDAX_NAMESPACE}} | grep size | cut -d':' -f2 | cut -d',' -f1)" - ctest -C ${{matrix.build_type}} -V -R devdax -E provider_devdax_memory_ipc + ctest -C ${{matrix.build_type}} -V -R devdax - # TODO: enable the provider_file_memory_ipc test when the IPC tests with the proxy library are fixed - # see the issue: https://github.com/oneapi-src/unified-memory-framework/issues/864 - name: Run the FSDAX tests with the proxy library # proxy library is built only if libumf is a shared library if: ${{ matrix.shared_library == 'ON' }} @@ -128,7 +125,13 @@ jobs: LD_PRELOAD=./lib/libumf_proxy.so UMF_TESTS_FSDAX_PATH=${{env.UMF_TESTS_FSDAX_PATH}} UMF_TESTS_FSDAX_PATH_2=${{env.UMF_TESTS_FSDAX_PATH_2}} - ctest -C ${{matrix.build_type}} -V -R "file|fsdax" -E provider_file_memory_ipc + ctest -C ${{matrix.build_type}} -V -R "file|fsdax" + + - name: Run DAX tests under valgrind + run: | + ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{env.BUILD_DIR}} memcheck "${{env.DAX_TESTS}}" + ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{env.BUILD_DIR}} drd "${{env.DAX_TESTS}}" + ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{env.BUILD_DIR}} helgrind "${{env.DAX_TESTS}}" - name: Check coverage if: ${{ matrix.build_type == 'Debug' }} diff --git a/.github/workflows/reusable_docs_build.yml b/.github/workflows/reusable_docs_build.yml index 269560c674..e90ca87aed 100644 --- a/.github/workflows/reusable_docs_build.yml +++ b/.github/workflows/reusable_docs_build.yml @@ -1,6 +1,12 @@ name: Docs build -on: workflow_call +on: + workflow_call: + inputs: + upload: + description: Should HTML documentation be uploaded as artifact? + type: boolean + default: false permissions: contents: read @@ -30,5 +36,17 @@ jobs: python3 -m pip install -r third_party/requirements.txt - name: Build the documentation - working-directory: scripts - run: python3 generate_docs.py + run: | + cmake -B build \ + -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF \ + -DUMF_BUILD_CUDA_PROVIDER=OFF \ + -DUMF_BUILD_TESTS=OFF \ + -DUMF_BUILD_EXAMPLES=OFF \ + -DUMF_DISABLE_HWLOC=ON + cmake --build build --target docs + + - name: Upload artifact + if: ${{ inputs.upload == true }} + uses: actions/upload-pages-artifact@0252fc4ba7626f0298f0cf00902a25c6afc77fa8 # v3.0.0 + with: + path: build/docs_build/generated/html diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml index e25de68a1b..58a172a74f 100644 --- a/.github/workflows/reusable_fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -79,19 +79,15 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt-get update - sudo apt-get install -y cmake libjemalloc-dev libhwloc-dev libnuma-dev libtbb-dev + sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev - name: Install dependencies (ubuntu-20.04) if: matrix.os == 'ubuntu-20.04' run: | sudo apt-get update - sudo apt-get install -y cmake libjemalloc-dev libnuma-dev libtbb-dev + sudo apt-get install -y cmake libnuma-dev libtbb-dev .github/scripts/install_hwloc.sh # install hwloc-2.3.0 instead of hwloc-2.1.0 present in the OS package - - name: Set ptrace value for IPC test (on Linux only) - if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-20.04' }} - run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" - - name: Configure CMake if: matrix.simple_cmake == 'OFF' run: > diff --git a/.github/workflows/reusable_multi_numa.yml b/.github/workflows/reusable_multi_numa.yml index 8b30ed53ed..f546b05451 100644 --- a/.github/workflows/reusable_multi_numa.yml +++ b/.github/workflows/reusable_multi_numa.yml @@ -10,6 +10,7 @@ env: BUILD_DIR : "${{github.workspace}}/build" COVERAGE_DIR : "${{github.workspace}}/coverage" COVERAGE_NAME : "exports-coverage-multinuma" + NUMA_TESTS: "./test/umf_test-memspace_numa ./test/umf_test-provider_os_memory_multiple_numa_nodes" jobs: multi_numa: @@ -45,7 +46,7 @@ jobs: -DUMF_BUILD_TESTS=ON -DUMF_DEVELOPER_MODE=ON -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON - -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON + -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=${{ matrix.os == 'rhel-9.1' && 'OFF' || 'ON' }} -DUMF_TESTS_FAIL_ON_SKIP=ON ${{ matrix.build_type == 'Debug' && matrix.os == 'ubuntu-22.04' && '-DUMF_USE_COVERAGE=ON' || '' }} @@ -68,6 +69,13 @@ jobs: ./test/umf_test-provider_os_memory_multiple_numa_nodes \ --gtest_filter="-*checkModeLocal/*:*checkModePreferredEmptyNodeset/*:testNuma.checkModeInterleave" + - name: Run NUMA tests under valgrind + if: matrix.os != 'rhel-9.1' + run: | + ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{env.BUILD_DIR}} memcheck "${{env.NUMA_TESTS}}" + ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{env.BUILD_DIR}} drd "${{env.NUMA_TESTS}}" + ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{env.BUILD_DIR}} helgrind "${{env.NUMA_TESTS}}" + - name: Check coverage if: ${{ matrix.build_type == 'Debug' && matrix.os == 'ubuntu-22.04' }} working-directory: ${{env.BUILD_DIR}} diff --git a/.github/workflows/reusable_proxy_lib.yml b/.github/workflows/reusable_proxy_lib.yml index 2a27161b3e..a1f5975fa6 100644 --- a/.github/workflows/reusable_proxy_lib.yml +++ b/.github/workflows/reusable_proxy_lib.yml @@ -32,10 +32,7 @@ jobs: - name: Install apt packages run: | sudo apt-get update - sudo apt-get install -y cmake libhwloc-dev libjemalloc-dev libtbb-dev lcov - - - name: Set ptrace value for IPC test - run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" + sudo apt-get install -y cmake libhwloc-dev libtbb-dev lcov - name: Configure build run: > @@ -49,7 +46,7 @@ jobs: -DUMF_BUILD_BENCHMARKS=OFF -DUMF_BUILD_TESTS=ON -DUMF_FORMAT_CODE_STYLE=OFF - -DUMF_DEVELOPER_MODE=OFF + -DUMF_DEVELOPER_MODE=ON -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON -DUMF_TESTS_FAIL_ON_SKIP=ON @@ -59,11 +56,9 @@ jobs: - name: Build UMF run: cmake --build ${{env.BUILD_DIR}} -j $(nproc) - # TODO enable the provider_file_memory_ipc test when the IPC tests with the proxy library are fixed - # see the issue: https://github.com/oneapi-src/unified-memory-framework/issues/864 - name: Run "ctest --output-on-failure" with proxy library working-directory: ${{env.BUILD_DIR}} - run: LD_PRELOAD=./lib/libumf_proxy.so ctest --output-on-failure -E provider_file_memory_ipc + run: LD_PRELOAD=./lib/libumf_proxy.so ctest --output-on-failure - name: Run "./test/umf_test-memoryPool" with proxy library working-directory: ${{env.BUILD_DIR}} @@ -77,14 +72,12 @@ jobs: working-directory: ${{env.BUILD_DIR}} run: UMF_PROXY="page.disposition=shared-shm" LD_PRELOAD=./lib/libumf_proxy.so /usr/bin/date - # TODO enable the provider_file_memory_ipc test when the IPC tests with the proxy library are fixed - # see the issue: https://github.com/oneapi-src/unified-memory-framework/issues/864 - name: Run "ctest --output-on-failure" with proxy library and size.threshold=128 working-directory: ${{env.BUILD_DIR}} run: > UMF_PROXY="page.disposition=shared-shm;size.threshold=128" LD_PRELOAD=./lib/libumf_proxy.so - ctest --output-on-failure -E provider_file_memory_ipc + ctest --output-on-failure - name: Check coverage if: ${{ matrix.build_type == 'Debug' }} diff --git a/.github/workflows/reusable_sanitizers.yml b/.github/workflows/reusable_sanitizers.yml index 3acda6833e..25458da51f 100644 --- a/.github/workflows/reusable_sanitizers.yml +++ b/.github/workflows/reusable_sanitizers.yml @@ -29,7 +29,7 @@ jobs: - name: Install apt packages run: | sudo apt-get update - sudo apt-get install -y clang cmake libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev + sudo apt-get install -y clang cmake libhwloc-dev libnuma-dev libtbb-dev - name: Install oneAPI basekit if: matrix.compiler.cxx == 'icpx' @@ -40,10 +40,6 @@ jobs: sudo apt-get update sudo apt-get install -y intel-oneapi-ippcp-devel intel-oneapi-ipp-devel intel-oneapi-common-oneapi-vars intel-oneapi-compiler-dpcpp-cpp - - - name: Set ptrace value for IPC test - run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" - - name: Configure build run: > ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh &&' || ''}} @@ -77,7 +73,6 @@ jobs: ASAN_OPTIONS: allocator_may_return_null=1 TSAN_OPTIONS: allocator_may_return_null=1 run: | - ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh' || true }} ctest --output-on-failure windows-build: diff --git a/.github/workflows/reusable_valgrind.yml b/.github/workflows/reusable_valgrind.yml index 86ceb68c68..aba0e32605 100644 --- a/.github/workflows/reusable_valgrind.yml +++ b/.github/workflows/reusable_valgrind.yml @@ -1,4 +1,4 @@ -# Run tests with valgrind intstrumentation tools: memcheck, drd, helgrind +# Run tests with valgrind instrumentation tools: memcheck, drd, helgrind name: Valgrind on: workflow_call @@ -20,7 +20,7 @@ jobs: - name: Install apt packages run: | sudo apt-get update - sudo apt-get install -y cmake hwloc libhwloc-dev libjemalloc-dev libnuma-dev libtbb-dev valgrind + sudo apt-get install -y cmake hwloc libhwloc-dev libnuma-dev libtbb-dev valgrind - name: Configure CMake run: > diff --git a/.gitignore b/.gitignore index a1a488bc14..e177e395ef 100644 --- a/.gitignore +++ b/.gitignore @@ -58,7 +58,7 @@ __pycache__/ *.py[cod] # Generated docs -docs/ +docs_build/ # Build files /build*/ diff --git a/CMakeLists.txt b/CMakeLists.txt index f71ce18201..70ac087991 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2024 Intel Corporation +# Copyright (C) 2022-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -11,7 +11,7 @@ list(APPEND CMAKE_MODULE_PATH "${UMF_CMAKE_SOURCE_DIR}/cmake") include(${UMF_CMAKE_SOURCE_DIR}/cmake/helpers.cmake) # We use semver aligned version, set via git tags. We parse git output to -# establih the version of UMF to be used in CMake, Win dll's, and within the +# establish the version of UMF to be used in CMake, Win dll's, and within the # code (e.g. in logger). We have 3-component releases (e.g. 1.5.1) plus release # candidates and git info. Function below sets all variables related to version. set_version_variables() @@ -65,7 +65,7 @@ umf_option( OFF) umf_option( UMF_LINK_HWLOC_STATICALLY - "Link UMF with HWLOC library statically (supported for Linux, MacOS and Release build on Windows)" + "Link UMF with HWLOC library statically (proxy library will be disabled on Windows+Debug build)" OFF) umf_option( UMF_FORMAT_CODE_STYLE @@ -123,6 +123,105 @@ else() message(FATAL_ERROR "Unknown OS type") endif() +if(UMF_DEVELOPER_MODE) + set(UMF_COMMON_COMPILE_DEFINITIONS ${UMF_COMMON_COMPILE_DEFINITIONS} + UMF_DEVELOPER_MODE=1) +endif() + +if(NOT UMF_BUILD_LIBUMF_POOL_JEMALLOC) + set(UMF_POOL_JEMALLOC_ENABLED FALSE) + set(JEMALLOC_FOUND FALSE) + set(JEMALLOC_LIBRARIES FALSE) +elseif(WINDOWS) + pkg_check_modules(JEMALLOC jemalloc) + if(NOT JEMALLOC_FOUND) + find_package(JEMALLOC REQUIRED jemalloc) + endif() +else() + if(NOT DEFINED UMF_JEMALLOC_REPO) + set(UMF_JEMALLOC_REPO "https://github.com/jemalloc/jemalloc.git") + endif() + + if(NOT DEFINED UMF_JEMALLOC_TAG) + set(UMF_JEMALLOC_TAG 5.3.0) + endif() + + include(FetchContent) + message( + STATUS + "Will fetch jemalloc from ${UMF_JEMALLOC_REPO} (tag: ${UMF_JEMALLOC_TAG})" + ) + + FetchContent_Declare( + jemalloc_targ + GIT_REPOSITORY ${UMF_JEMALLOC_REPO} + GIT_TAG ${UMF_JEMALLOC_TAG}) + FetchContent_MakeAvailable(jemalloc_targ) + + add_custom_command( + COMMAND ./autogen.sh + WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR} + OUTPUT ${jemalloc_targ_SOURCE_DIR}/configure) + add_custom_command( + # Custom jemalloc build. Non-default options used: + # --with-jemalloc-prefix=je_ - add je_ prefix to all public APIs + # --disable-cxx - Disable C++ integration. This will cause new and + # delete operators implementations to be omitted. + # --disable-initial-exec-tls - Disable the initial-exec TLS model for + # jemalloc's internal thread-local storage (on those platforms that + # support explicit settings). This can allow jemalloc to be dynamically + # loaded after program startup (e.g. using dlopen). + COMMAND + ./configure --prefix=${jemalloc_targ_BINARY_DIR} + --with-jemalloc-prefix=je_ --disable-cxx --disable-initial-exec-tls + CFLAGS=-fPIC + WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR} + OUTPUT ${jemalloc_targ_SOURCE_DIR}/Makefile + DEPENDS ${jemalloc_targ_SOURCE_DIR}/configure) + + if(NOT UMF_QEMU_BUILD) + set(MAKE_ARGUMENTS "-j$(nproc)") + endif() + + add_custom_command( + COMMAND make ${MAKE_ARGUMENTS} + WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR} + OUTPUT ${jemalloc_targ_SOURCE_DIR}/lib/libjemalloc.a + DEPENDS ${jemalloc_targ_SOURCE_DIR}/Makefile) + add_custom_command( + COMMAND make install + WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR} + OUTPUT ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a + DEPENDS ${jemalloc_targ_SOURCE_DIR}/lib/libjemalloc.a) + + add_custom_target(jemalloc_prod + DEPENDS ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a) + add_library(jemalloc INTERFACE) + target_link_libraries( + jemalloc INTERFACE ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a) + add_dependencies(jemalloc jemalloc_prod) + + set(JEMALLOC_LIBRARY_DIRS ${jemalloc_targ_BINARY_DIR}/lib) + set(JEMALLOC_INCLUDE_DIRS ${jemalloc_targ_BINARY_DIR}/include) + set(JEMALLOC_LIBRARIES ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a) +endif() + +if(JEMALLOC_FOUND OR JEMALLOC_LIBRARIES) + set(UMF_POOL_JEMALLOC_ENABLED TRUE) + # add PATH to DLL on Windows + set(DLL_PATH_LIST + "${DLL_PATH_LIST};PATH=path_list_append:${JEMALLOC_DLL_DIRS}") + message(STATUS " JEMALLOC_LIBRARIES = ${JEMALLOC_LIBRARIES}") + message(STATUS " JEMALLOC_INCLUDE_DIRS = ${JEMALLOC_INCLUDE_DIRS}") + message(STATUS " JEMALLOC_LIBRARY_DIRS = ${JEMALLOC_LIBRARY_DIRS}") +else() + set(UMF_POOL_JEMALLOC_ENABLED FALSE) + message( + STATUS + "Disabling the Jemalloc Pool and tests and benchmarks that use it because jemalloc was not built/found." + ) +endif() + if(UMF_DISABLE_HWLOC) message(STATUS "hwloc is disabled, hence OS provider, memtargets, " "topology discovery, examples won't be available!") @@ -164,8 +263,8 @@ else() set(LIBHWLOC_INCLUDE_DIRS ${hwloc_targ_SOURCE_DIR}/include;${hwloc_targ_BINARY_DIR}/include) - set(LIBHWLOC_LIBRARY_DIRS - ${hwloc_targ_BINARY_DIR}/Release;${hwloc_targ_BINARY_DIR}/Debug) + set(LIBHWLOC_LIBRARY_DIRS ${hwloc_targ_BINARY_DIR}/$) + set(LIBHWLOC_LIBRARIES ${hwloc_targ_BINARY_DIR}/$/hwloc.lib) else() include(FetchContent) message( @@ -414,19 +513,6 @@ else() set(UMF_POOL_SCALABLE_ENABLED FALSE) endif() -if(UMF_BUILD_LIBUMF_POOL_JEMALLOC) - pkg_check_modules(JEMALLOC jemalloc) - if(NOT JEMALLOC_FOUND) - find_package(JEMALLOC REQUIRED jemalloc) - endif() - if(JEMALLOC_FOUND OR JEMALLOC_LIBRARIES) - set(UMF_POOL_JEMALLOC_ENABLED TRUE) - # add PATH to DLL on Windows - set(DLL_PATH_LIST - "${DLL_PATH_LIST};PATH=path_list_append:${JEMALLOC_DLL_DIRS}") - endif() -endif() - if(WINDOWS) # TODO: enable the proxy library in the Debug build on Windows # @@ -467,14 +553,14 @@ elseif(UMF_PROXY_LIB_BASED_ON_POOL STREQUAL SCALABLE) ) endif() elseif(UMF_PROXY_LIB_BASED_ON_POOL STREQUAL JEMALLOC) - if(UMF_BUILD_LIBUMF_POOL_JEMALLOC) + if(UMF_POOL_JEMALLOC_ENABLED) set(UMF_PROXY_LIB_ENABLED ON) set(PROXY_LIB_USES_JEMALLOC_POOL ON) - set(PROXY_LIBS jemalloc_pool umf) + set(PROXY_LIBS umf) else() message( STATUS - "Disabling the proxy library, because UMF_PROXY_LIB_BASED_ON_POOL==JEMALLOC but UMF_BUILD_LIBUMF_POOL_JEMALLOC is OFF" + "Disabling the proxy library, because UMF_PROXY_LIB_BASED_ON_POOL==JEMALLOC but the jemalloc pool is disabled" ) endif() else() @@ -686,6 +772,17 @@ if(UMF_FORMAT_CODE_STYLE) endif() endif() +find_package(Python3 3.8) +if(Python3_FOUND) + message(STATUS "Adding 'docs' target for creating a documentation.") + add_custom_target( + docs + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMAND UMF_VERSION=${UMF_CMAKE_VERSION} ${Python3_EXECUTABLE} + ${UMF_CMAKE_SOURCE_DIR}/docs/generate_docs.py + COMMENT "Generate HTML documentation using Doxygen") +endif() + # --------------------------------------------------------------------------- # # Configure make install/uninstall and packages # --------------------------------------------------------------------------- # diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 58dba18db6..2e7fbf7d6c 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -5,7 +5,7 @@ We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, +identity and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation. diff --git a/ChangeLog b/ChangeLog index 0736379f85..a4afa52cee 100644 --- a/ChangeLog +++ b/ChangeLog @@ -31,6 +31,12 @@ Mon Dec 09 2024 Łukasz Stolarczuk - extended logging - yet more fixes in the building system +Tue Nov 12 2024 Łukasz Stolarczuk + + * Version 0.9.1 + + This patch release contains only 3 small fixes in build system of UMF. + Thu Sep 12 2024 Łukasz Stolarczuk * Version 0.9.0 diff --git a/README.md b/README.md index 6f1233c639..5bd0b9b2f5 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ For a quick introduction to UMF usage, please see [examples](https://oneapi-src.github.io/unified-memory-framework/examples.html) documentation, which includes the code of the [basic example](https://github.com/oneapi-src/unified-memory-framework/blob/main/examples/basic/basic.c). -The are also more advanced that allocates USM memory from the +The are also more advanced that allocates USM memory from the [Level Zero device](https://github.com/oneapi-src/unified-memory-framework/blob/main/examples/level_zero_shared_memory/level_zero_shared_memory.c) using the Level Zero API and UMF Level Zero memory provider and [CUDA device](https://github.com/oneapi-src/unified-memory-framework/blob/main/examples/cuda_shared_memory/cuda_shared_memory.c) using the CUDA API and UMF CUDA memory provider. @@ -28,19 +28,23 @@ using the CUDA API and UMF CUDA memory provider. ### Requirements Required packages: + - libhwloc-dev >= 2.3.0 (Linux) / hwloc >= 2.3.0 (Windows) - C compiler - [CMake](https://cmake.org/) >= 3.14.0 For development and contributions: + - clang-format-15.0 (can be installed with `python -m pip install clang-format==15.0.7`) - cmake-format-0.6 (can be installed with `python -m pip install cmake-format==0.6.13`) - black (can be installed with `python -m pip install black==24.3.0`) For building tests, multithreaded benchmarks and Disjoint Pool: + - C++ compiler with C++17 support For Level Zero memory provider tests: + - Level Zero headers and libraries - compatible GPU with installed driver @@ -50,8 +54,8 @@ Executable and binaries will be in **build/bin**. The `{build_config}` can be either `Debug` or `Release`. ```bash -$ cmake -B build -DCMAKE_BUILD_TYPE={build_config} -$ cmake --build build -j $(nproc) +cmake -B build -DCMAKE_BUILD_TYPE={build_config} +cmake --build build -j $(nproc) ``` ### Windows @@ -60,8 +64,8 @@ Generating Visual Studio Project. EXE and binaries will be in **build/bin/{build The `{build_config}` can be either `Debug` or `Release`. ```bash -$ cmake -B build -G "Visual Studio 15 2017 Win64" -$ cmake --build build --config {build_config} -j $Env:NUMBER_OF_PROCESSORS +cmake -B build -G "Visual Studio 15 2017 Win64" +cmake --build build --config {build_config} -j $Env:NUMBER_OF_PROCESSORS ``` ### Benchmark @@ -73,20 +77,22 @@ UMF also provides multithreaded benchmarks that can be enabled by setting both `UMF_BUILD_BENCHMARKS` and `UMF_BUILD_BENCHMARKS_MT` CMake configuration flags to `ON`. Multithreaded benchmarks require a C++ support. -The Scalable Pool requirements can be found in the relevant 'Memory Pool +The Scalable Pool requirements can be found in the relevant 'Memory Pool managers' section below. ### Sanitizers List of sanitizers available on Linux: + - AddressSanitizer - UndefinedBehaviorSanitizer - ThreadSanitizer - - Is mutually exclusive with other sanitizers. + - Is mutually exclusive with other sanitizers. - MemorySanitizer - - Requires linking against MSan-instrumented libraries to prevent false positive reports. More information [here](https://github.com/google/sanitizers/wiki/MemorySanitizerLibcxxHowTo). + - Requires linking against MSan-instrumented libraries to prevent false positive reports. More information [here](https://github.com/google/sanitizers/wiki/MemorySanitizerLibcxxHowTo). List of sanitizers available on Windows: + - AddressSanitizer Listed sanitizers can be enabled with appropriate [CMake options](#cmake-standard-options). @@ -117,81 +123,107 @@ List of options provided by CMake: | UMF_USE_MSAN | Enable MemorySanitizer checks | ON/OFF | OFF | | UMF_USE_VALGRIND | Enable Valgrind instrumentation | ON/OFF | OFF | | UMF_USE_COVERAGE | Build with coverage enabled (Linux only) | ON/OFF | OFF | -| UMF_LINK_HWLOC_STATICALLY | Link UMF with HWLOC library statically (Windows+Release only) | ON/OFF | OFF | +| UMF_LINK_HWLOC_STATICALLY | Link UMF with HWLOC library statically (proxy library will be disabled on Windows+Debug build) | ON/OFF | OFF | | UMF_DISABLE_HWLOC | Disable features that requires hwloc (OS provider, memory targets, topology discovery) | ON/OFF | OFF | ## Architecture: memory pools and providers -A UMF memory pool is a combination of a pool allocator and a memory provider. A memory provider is responsible for coarse-grained memory allocations and management of memory pages, while the pool allocator controls memory pooling and handles fine-grained memory allocations. +A UMF memory pool is a combination of a pool allocator and a memory provider. A memory provider is responsible for +coarse-grained memory allocations and management of memory pages, while the pool allocator controls memory pooling +and handles fine-grained memory allocations. Pool allocator can leverage existing allocators (e.g. jemalloc or tbbmalloc) or be written from scratch. -UMF comes with predefined pool allocators (see include/pool) and providers (see include/provider). UMF can also work with user-defined pools and providers that implement a specific interface (see include/umf/memory_pool_ops.h and include/umf/memory_provider_ops.h). +UMF comes with predefined pool allocators (see [`include/umf/pools`](include/umf/pools)) and providers +(see [`include/umf/providers`](include/umf/providers)). UMF can also work with user-defined pools and +providers that implement a specific interface (see [`include/umf/memory_pool_ops.h`](include/umf/memory_pool_ops.h) +and [`include/umf/memory_provider_ops.h`](include/umf/memory_provider_ops.h)). -More detailed documentation is available here: https://oneapi-src.github.io/unified-memory-framework/ +More detailed documentation is available here: ### Memory providers -#### Coarse Provider +#### Fixed memory provider -A memory provider that can provide memory from: -1) a given pre-allocated buffer (the fixed-size memory provider option) or -2) from an additional upstream provider (e.g. provider that does not support the free() operation - like the File memory provider or the DevDax memory provider - see below). +A memory provider that can provide memory from a given pre-allocated buffer. #### OS memory provider A memory provider that provides memory from an operating system. OS memory provider supports two types of memory mappings (set by the `visibility` parameter): + 1) private memory mapping (`UMF_MEM_MAP_PRIVATE`) 2) shared memory mapping (`UMF_MEM_MAP_SHARED` - supported on Linux only yet) IPC API requires the `UMF_MEM_MAP_SHARED` memory `visibility` mode (`UMF_RESULT_ERROR_INVALID_ARGUMENT` is returned otherwise). +IPC API uses file descriptor duplication, which requires the `pidfd_getfd(2)` system call to obtain +a duplicate of another process's file descriptor. This system call is supported since Linux 5.6. +Required permission ("restricted ptrace") is governed by the `PTRACE_MODE_ATTACH_REALCREDS` check +(see `ptrace(2)`). To allow file descriptor duplication in a binary that opens IPC handle, you can call +`prctl(PR_SET_PTRACER, ...)` in the producer binary that gets the IPC handle. +Alternatively you can change the `ptrace_scope` globally in the system, e.g.: + +```sh +sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" +``` + There are available two mechanisms for the shared memory mapping: + 1) a named shared memory object (used if the `shm_name` parameter is not NULL) or 2) an anonymous file descriptor (used if the `shm_name` parameter is NULL) The `shm_name` parameter should be a null-terminated string of up to NAME_MAX (i.e., 255) characters none of which are slashes. An anonymous file descriptor for the shared memory mapping will be created using: + 1) `memfd_secret()` syscall - (if it is implemented and) if the `UMF_MEM_FD_FUNC` environment variable does not contain the "memfd_create" string or 2) `memfd_create()` syscall - otherwise (and if it is implemented). ##### Requirements -Required packages for tests (Linux-only yet): - - libnuma-dev +IPC API on Linux requires the `PTRACE_MODE_ATTACH_REALCREDS` permission (see `ptrace(2)`) +to duplicate another process's file descriptor (see above). + +Packages required for tests (Linux-only yet): + +- libnuma-dev #### Level Zero memory provider A memory provider that provides memory from L0 device. +IPC API uses file descriptor duplication, which requires the `pidfd_getfd(2)` system call to obtain +a duplicate of another process's file descriptor. This system call is supported since Linux 5.6. +Required permission ("restricted ptrace") is governed by the `PTRACE_MODE_ATTACH_REALCREDS` check +(see `ptrace(2)`). To allow file descriptor duplication in a binary that opens IPC handle, you can call +`prctl(PR_SET_PTRACER, ...)` in the producer binary that gets the IPC handle. +Alternatively you can change the `ptrace_scope` globally in the system, e.g.: + +```sh +sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" +``` ##### Requirements 1) Linux or Windows OS 2) The `UMF_BUILD_LEVEL_ZERO_PROVIDER` option turned `ON` (by default) +3) IPC API on Linux requires the `PTRACE_MODE_ATTACH_REALCREDS` permission (see `ptrace(2)`) + to duplicate another process's file descriptor (see above). Additionally, required for tests: -3) The `UMF_BUILD_GPU_TESTS` option turned `ON` -4) System with Level Zero compatible GPU -5) Required packages: +4) The `UMF_BUILD_GPU_TESTS` option turned `ON` +5) System with Level Zero compatible GPU +6) Required packages: - liblevel-zero-dev (Linux) or level-zero-sdk (Windows) #### DevDax memory provider (Linux only) -A memory provider that provides memory from a device DAX (a character device file /dev/daxX.Y). +A memory provider that provides memory from a device DAX (a character device file like `/dev/daxX.Y`). It can be used when large memory mappings are needed. -The DevDax memory provider does not support the free operation -(`umfMemoryProviderFree()` always returns `UMF_RESULT_ERROR_NOT_SUPPORTED`), -so it should be used with a pool manager that will take over -the managing of the provided memory - for example the jemalloc pool -with the `disable_provider_free` parameter set to true. - ##### Requirements 1) Linux OS @@ -201,12 +233,6 @@ with the `disable_provider_free` parameter set to true. A memory provider that provides memory by mapping a regular, extendable file. -The file memory provider does not support the free operation -(`umfMemoryProviderFree()` always returns `UMF_RESULT_ERROR_NOT_SUPPORTED`), -so it should be used with a pool manager that will take over -the managing of the provided memory - for example the jemalloc pool -with the `disable_provider_free` parameter set to true. - IPC API requires the `UMF_MEM_MAP_SHARED` memory `visibility` mode (`UMF_RESULT_ERROR_INVALID_ARGUMENT` is returned otherwise). @@ -241,8 +267,6 @@ This memory pool is distributed as part of libumf. It forwards all requests to t memory provider. Currently umfPoolRealloc, umfPoolCalloc and umfPoolMallocUsableSize functions are not supported by the proxy pool. -To enable this feature, the `UMF_BUILD_SHARED_LIBRARY` option needs to be turned `ON`. - #### Disjoint pool TODO: Add a description @@ -253,16 +277,33 @@ To enable this feature, the `UMF_BUILD_LIBUMF_POOL_DISJOINT` option needs to be #### Jemalloc pool -Jemalloc pool is a [jemalloc](https://github.com/jemalloc/jemalloc)-based memory +Jemalloc pool is a [jemalloc](https://github.com/jemalloc/jemalloc)-based memory pool manager built as a separate static library: libjemalloc_pool.a on Linux and jemalloc_pool.lib on Windows. The `UMF_BUILD_LIBUMF_POOL_JEMALLOC` option has to be turned `ON` to build this library. +[jemalloc](https://github.com/jemalloc/jemalloc) is required to build the jemalloc pool. + +In case of Linux OS jemalloc is built from the (fetched) sources with the following +non-default options enabled: + +- `--with-jemalloc-prefix=je_` - adds the `je_` prefix to all public APIs, +- `--disable-cxx` - disables C++ integration, it will cause the `new` and the `delete` + operators implementations to be omitted. +- `--disable-initial-exec-tls` - disables the initial-exec TLS model for jemalloc's + internal thread-local storage (on those platforms that support + explicit settings), it can allow jemalloc to be dynamically + loaded after program startup (e.g. using `dlopen()`). + +The default jemalloc package is required on Windows. + ##### Requirements 1) The `UMF_BUILD_LIBUMF_POOL_JEMALLOC` option turned `ON` -2) Required packages: - - libjemalloc-dev (Linux) or jemalloc (Windows) +2) jemalloc is required: + +- on Linux and MacOS: jemalloc is fetched and built from sources (a custom build), +- on Windows: the default jemalloc package is required #### Scalable Pool (part of libumf) @@ -272,7 +313,8 @@ It is distributed as part of libumf. To use this pool, TBB must be installed in ##### Requirements Packages required for using this pool and executing tests/benchmarks (not required for build): - - libtbb-dev (libtbbmalloc.so.2) on Linux or tbb (tbbmalloc.dll) on Windows + +- libtbb-dev (libtbbmalloc.so.2) on Linux or tbb (tbbmalloc.dll) on Windows ### Memspaces (Linux-only) @@ -303,19 +345,22 @@ Querying the latency value requires HMAT support on the platform. Calling `umfMe UMF provides the UMF proxy library (`umf_proxy`) that makes it possible to override the default allocator in other programs in both Linux and Windows. +To enable this feature, the `UMF_BUILD_SHARED_LIBRARY` option needs to be turned `ON`. + #### Linux In case of Linux it can be done without any code changes using the `LD_PRELOAD` environment variable: ```sh -$ LD_PRELOAD=/usr/lib/libumf_proxy.so myprogram +LD_PRELOAD=/usr/lib/libumf_proxy.so myprogram ``` The memory used by the proxy memory allocator is mmap'ed: + 1) with the `MAP_PRIVATE` flag by default or 2) with the `MAP_SHARED` flag if the `UMF_PROXY` environment variable contains one of two following strings: `page.disposition=shared-shm` or `page.disposition=shared-fd`. These two options differ in a mechanism used during IPC: - `page.disposition=shared-shm` - IPC uses the named shared memory. An SHM name is generated using the `umf_proxy_lib_shm_pid_$PID` pattern, where `$PID` is the PID of the process. It creates the `/dev/shm/umf_proxy_lib_shm_pid_$PID` file. - - `page.disposition=shared-fd` - IPC uses the file descriptor duplication. It requires using `pidfd_getfd(2)` to obtain a duplicate of another process's file descriptor. Permission to duplicate another process's file descriptor is governed by a ptrace access mode `PTRACE_MODE_ATTACH_REALCREDS` check (see `ptrace(2)`) that can be changed using the `/proc/sys/kernel/yama/ptrace_scope` interface. `pidfd_getfd(2)` is supported since Linux 5.6. + - `page.disposition=shared-fd` - IPC API uses file descriptor duplication, which requires the `pidfd_getfd(2)` system call to obtain a duplicate of another process's file descriptor. This system call is supported since Linux 5.6. Required permission ("restricted ptrace") is governed by the `PTRACE_MODE_ATTACH_REALCREDS` check (see `ptrace(2)`). To allow file descriptor duplication in a binary that opens IPC handle, you can call `prctl(PR_SET_PTRACER, ...)` in the producer binary that gets the IPC handle. Alternatively you can change the `ptrace_scope` globally in the system, e.g.: `sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"`. **Size threshold** @@ -327,6 +372,7 @@ It can be enabled by adding the `size.threshold=` string to the `UMF_PROX #### Windows In case of Windows it requires: + 1) explicitly linking your program dynamically with the `umf_proxy.dll` library 2) (C++ code only) including `proxy_lib_new_delete.h` in a single(!) source file in your project to override also the `new`/`delete` operations. @@ -340,3 +386,7 @@ an issue or a Pull Request, please read [Contribution Guide](./CONTRIBUTING.md). To enable logging in UMF source files please follow the guide in the [web documentation](https://oneapi-src.github.io/unified-memory-framework/introduction.html#logging). + +## Notices + +The contents of this repository may have been developed with support from one or more Intel-operated generative artificial intelligence solutions. diff --git a/RELEASE_STEPS.md b/RELEASE_STEPS.md index ec6e5b6906..09a972598b 100644 --- a/RELEASE_STEPS.md +++ b/RELEASE_STEPS.md @@ -39,10 +39,7 @@ Do changes for a release: - For major/minor release start from the `main` branch - Add an entry to ChangeLog, remember to change the day of the week in the release date - For major and minor (prior 1.0.0) releases mention API and ABI compatibility with the previous release -- Update project's version in a few places: - - For major and minor releases: `UMF_VERSION_CURRENT` in `include/umf/base.h` (the API version) - - `release` variable in `scripts/docs_config/conf.py` (for docs) - - `UMF_VERSION` variable in `.github/workflows/basic.yml` (for installation test) +- For major and minor releases, update `UMF_VERSION_CURRENT` in `include/umf/base.h` (the API version) - For major and minor (prior 1.0.0) releases update ABI version in `.map` and `.def` files - These files are defined for all public libraries (`libumf` and `proxy_lib`, at the moment) - Commit these changes and tag the release: diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 5605519ee2..efad0baf36 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -51,7 +51,7 @@ function(add_umf_benchmark) set(BENCH_NAME umf-${ARG_NAME}) - set(BENCH_LIBS ${ARG_LIBS} umf) + set(BENCH_LIBS ${ARG_LIBS} umf umf_utils) add_umf_executable( NAME ${BENCH_NAME} @@ -121,10 +121,6 @@ set(LIB_DIRS ${LIBHWLOC_LIBRARY_DIRS}) if(UMF_BUILD_LIBUMF_POOL_DISJOINT) set(LIBS_OPTIONAL ${LIBS_OPTIONAL} disjoint_pool) endif() -if(UMF_BUILD_LIBUMF_POOL_JEMALLOC) - set(LIBS_OPTIONAL ${LIBS_OPTIONAL} jemalloc_pool ${JEMALLOC_LIBRARIES}) - set(LIB_DIRS ${LIB_DIRS} ${JEMALLOC_LIBRARY_DIRS}) -endif() if(LINUX) set(LIBS_OPTIONAL ${LIBS_OPTIONAL} m) endif() diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index c10bbda877..6c8175e1d0 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -7,223 +7,78 @@ */ #include -#include -#ifdef UMF_POOL_SCALABLE_ENABLED -#include -#endif -#include - -#ifdef UMF_POOL_DISJOINT_ENABLED -#include -#endif - -#ifdef UMF_POOL_JEMALLOC_ENABLED -#include -#endif #include "benchmark.hpp" -struct glibc_malloc : public allocator_interface { - unsigned SetUp([[maybe_unused]] ::benchmark::State &state, - unsigned argPos) override { - return argPos; - } - void TearDown([[maybe_unused]] ::benchmark::State &state) override{}; - void *benchAlloc(size_t size) override { return malloc(size); } - void benchFree(void *ptr, [[maybe_unused]] size_t size) override { - free(ptr); - } - static std::string name() { return "glibc"; } -}; - -struct os_provider : public provider_interface { - umf_os_memory_provider_params_handle_t params = NULL; - os_provider() { - umfOsMemoryProviderParamsCreate(¶ms); - return; +#define UMF_BENCHMARK_TEMPLATE_DEFINE(BaseClass, Method, ...) \ + BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, __VA_ARGS__) \ + (benchmark::State & state) { \ + for (auto _ : state) { \ + bench(state); \ + } \ } - ~os_provider() { - if (params != NULL) { - umfOsMemoryProviderParamsDestroy(params); - } - } +#define UMF_BENCHMARK_REGISTER_F(BaseClass, Method) \ + BENCHMARK_REGISTER_F(BaseClass, Method) \ + ->Apply( \ + &BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::defaultArgs) - void *getParams() override { return params; } - umf_memory_provider_ops_t *getOps() override { - return umfOsMemoryProviderOps(); - } - static std::string name() { return "os_provider"; } -}; - -template -struct proxy_pool : public pool_interface { - umf_memory_pool_ops_t * - getOps([[maybe_unused]] ::benchmark::State &state) override { - return umfProxyPoolOps(); - } - void *getParams([[maybe_unused]] ::benchmark::State &state) override { - return nullptr; - } - static std::string name() { return "proxy_pool<" + Provider::name() + ">"; } -}; - -#ifdef UMF_POOL_DISJOINT_ENABLED -template -struct disjoint_pool : public pool_interface { - umf_disjoint_pool_params_handle_t disjoint_memory_pool_params; - - disjoint_pool() { - disjoint_memory_pool_params = NULL; - auto ret = umfDisjointPoolParamsCreate(&disjoint_memory_pool_params); - if (ret != UMF_RESULT_SUCCESS) { - return; - } - - // those function should never fail, so error handling is minimal. - ret = umfDisjointPoolParamsSetSlabMinSize(disjoint_memory_pool_params, - 4096); - if (ret != UMF_RESULT_SUCCESS) { - goto err; - } - - ret = umfDisjointPoolParamsSetCapacity(disjoint_memory_pool_params, 4); - if (ret != UMF_RESULT_SUCCESS) { - goto err; - } - - ret = umfDisjointPoolParamsSetMinBucketSize(disjoint_memory_pool_params, - 4096); - if (ret != UMF_RESULT_SUCCESS) { - goto err; - } - - ret = umfDisjointPoolParamsSetMaxPoolableSize( - disjoint_memory_pool_params, 4096 * 16); - - if (ret != UMF_RESULT_SUCCESS) { - goto err; - } - return; - err: - - umfDisjointPoolParamsDestroy(disjoint_memory_pool_params); - disjoint_memory_pool_params = NULL; - } - - ~disjoint_pool() { - if (disjoint_memory_pool_params != NULL) { - umfDisjointPoolParamsDestroy(disjoint_memory_pool_params); - } - } - - umf_memory_pool_ops_t * - getOps([[maybe_unused]] ::benchmark::State &state) override { - return umfDisjointPoolOps(); - } - void *getParams([[maybe_unused]] ::benchmark::State &state) override { - - if (disjoint_memory_pool_params == NULL) { - state.SkipWithError("Failed to create disjoint pool params"); - } +// Benchmarks scenarios: - return disjoint_memory_pool_params; - } - static std::string name() { - return "disjoint_pool<" + Provider::name() + ">"; - } -}; -#endif +// The benchmark arguments specified in Args() are, in order: +// benchmark arguments, allocator arguments, size generator arguments. +// The exact meaning of each argument depends on the benchmark, allocator, and size components used. +// Refer to the 'argsName()' function in each component to find detailed descriptions of these arguments. -#ifdef UMF_POOL_JEMALLOC_ENABLED -template -struct jemalloc_pool : public pool_interface { - umf_memory_pool_ops_t * - getOps([[maybe_unused]] ::benchmark::State &state) override { - return umfJemallocPoolOps(); - } - void *getParams([[maybe_unused]] ::benchmark::State &state) override { - return NULL; - } - static std::string name() { - return "jemalloc_pool<" + Provider::name() + ">"; - } -}; -#endif +static void default_alloc_fix_size(benchmark::internal::Benchmark *benchmark) { + benchmark->Args({10000, 0, 4096}); + benchmark->Args({10000, 100000, 4096}); + benchmark->Threads(4); + benchmark->Threads(1); +} -#ifdef UMF_POOL_SCALABLE_ENABLED -template -struct scalable_pool : public pool_interface { - virtual umf_memory_pool_ops_t * - getOps([[maybe_unused]] ::benchmark::State &state) override { - return umfScalablePoolOps(); - } - virtual void * - getParams([[maybe_unused]] ::benchmark::State &state) override { - return NULL; - } - static std::string name() { - return "scalable_pool<" + Provider::name() + ">"; - } -}; -#endif -// Benchmarks scenarios: +static void +default_alloc_uniform_size(benchmark::internal::Benchmark *benchmark) { + benchmark->Args({10000, 0, 8, 64 * 1024, 8}); + benchmark->Threads(4); + benchmark->Threads(1); +} UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, glibc_fix, fixed_alloc_size, glibc_malloc); -// The benchmark arguments specified in Args() are, in order: -// benchmark arguments, allocator arguments, size generator arguments. -// The exact meaning of each argument depends on the benchmark, allocator, and size components used. -// Refer to the 'argsName()' function in each component to find detailed descriptions of these arguments. UMF_BENCHMARK_REGISTER_F(alloc_benchmark, glibc_fix) - ->Args({10000, 0, 4096}) - ->Args({10000, 100000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_fix_size); UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, glibc_uniform, uniform_alloc_size, glibc_malloc); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, glibc_uniform) - ->Args({10000, 0, 8, 64 * 1024, 8}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_uniform_size); UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, os_provider, fixed_alloc_size, provider_allocator); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, os_provider) - ->Args({10000, 0, 4096}) - ->Args({10000, 100000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_fix_size); UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, proxy_pool, fixed_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, proxy_pool) - ->Args({1000, 0, 4096}) - ->Args({1000, 100000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_fix_size); #ifdef UMF_POOL_DISJOINT_ENABLED UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, disjoint_pool_fix, fixed_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, disjoint_pool_fix) - ->Args({10000, 0, 4096}) - ->Args({10000, 100000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_fix_size); // TODO: debug why this crashes /*UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, disjoint_pool_uniform, uniform_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, disjoint_pool_uniform) - ->Args({10000, 0, 8, 64 * 1024, 8}) - // ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_uniform_size); */ #endif @@ -232,18 +87,13 @@ UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, jemalloc_pool_fix, fixed_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, jemalloc_pool_fix) - ->Args({10000, 0, 4096}) - ->Args({10000, 100000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_fix_size); UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, jemalloc_pool_uniform, uniform_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, jemalloc_pool_uniform) - ->Args({10000, 0, 8, 64 * 1024, 8}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_uniform_size); #endif #ifdef UMF_POOL_SCALABLE_ENABLED @@ -252,71 +102,67 @@ UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, scalable_pool_fix, pool_allocator>); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, scalable_pool_fix) - ->Args({10000, 0, 4096}) - ->Args({10000, 100000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_fix_size); UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, scalable_pool_uniform, uniform_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, scalable_pool_uniform) - ->Args({10000, 0, 8, 64 * 1024, 8}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_uniform_size); #endif // Multiple allocs/free +static void +default_multiple_alloc_fix_size(benchmark::internal::Benchmark *benchmark) { + benchmark->Args({10000, 4096}); + benchmark->Threads(4); + benchmark->Threads(1); +} + +static void +default_multiple_alloc_uniform_size(benchmark::internal::Benchmark *benchmark) { + benchmark->Args({10000, 8, 64 * 1024, 8}); + benchmark->Threads(4); + benchmark->Threads(1); +} UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, glibc_fix, fixed_alloc_size, glibc_malloc); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, glibc_fix) - ->Args({10000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_fix_size); UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, glibc_uniform, uniform_alloc_size, glibc_malloc); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, glibc_uniform) - ->Args({10000, 8, 64 * 1024, 8}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_uniform_size); UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, proxy_pool, fixed_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, proxy_pool) - ->Args({10000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_fix_size); UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, os_provider, fixed_alloc_size, provider_allocator); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, os_provider) - ->Args({10000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_fix_size); #ifdef UMF_POOL_DISJOINT_ENABLED UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, disjoint_pool_fix, fixed_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, disjoint_pool_fix) - ->Args({10000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_fix_size); // TODO: debug why this crashes /*UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, disjoint_pool_uniform, uniform_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, disjoint_pool_uniform) - ->Args({10000, 0, 8, 64 * 1024, 8}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_uniform_size); */ #endif @@ -325,17 +171,13 @@ UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, jemalloc_pool_fix, fixed_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, jemalloc_pool_fix) - ->Args({10000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_fix_size); UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, jemalloc_pool_uniform, uniform_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, jemalloc_pool_uniform) - ->Args({1000, 8, 64 * 1024, 8}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_uniform_size); #endif @@ -345,18 +187,14 @@ UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, scalable_pool_fix, pool_allocator>); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, scalable_pool_fix) - ->Args({10000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_fix_size); UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, scalable_pool_uniform, uniform_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, scalable_pool_uniform) - ->Args({10000, 8, 64 * 1024, 8}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_uniform_size); #endif BENCHMARK_MAIN(); diff --git a/benchmark/benchmark.hpp b/benchmark/benchmark.hpp index ead6b39e75..50e75f8fb2 100644 --- a/benchmark/benchmark.hpp +++ b/benchmark/benchmark.hpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -75,69 +75,104 @@ #include #include -#include "benchmark_interfaces.hpp" +#include "benchmark_size.hpp" +#include "benchmark_umf.hpp" struct alloc_data { void *ptr; size_t size; }; -#define UMF_BENCHMARK_TEMPLATE_DEFINE(BaseClass, Method, ...) \ - BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, __VA_ARGS__) \ - (benchmark::State & state) { \ - for (auto _ : state) { \ - bench(state); \ - } \ +template ::value>> +class provider_allocator : public allocator_interface { + public: + unsigned SetUp(::benchmark::State &state, unsigned r) override { + provider.SetUp(state); + return r; } -#define UMF_BENCHMARK_REGISTER_F(BaseClass, Method) \ - BENCHMARK_REGISTER_F(BaseClass, Method) \ - ->ArgNames( \ - BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::argsName()) \ - ->Name(BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::name()) \ - ->MinWarmUpTime(1) + void TearDown(::benchmark::State &state) override { + provider.TearDown(state); + } -class fixed_alloc_size : public alloc_size_interface { - public: - unsigned SetUp(::benchmark::State &state, unsigned argPos) override { - size = state.range(argPos); - return argPos + 1; + void *benchAlloc(size_t size) override { + void *ptr; + if (umfMemoryProviderAlloc(provider.provider, size, 0, &ptr) != + UMF_RESULT_SUCCESS) { + return NULL; + } + return ptr; + } + + void benchFree(void *ptr, size_t size) override { + umfMemoryProviderFree(provider.provider, ptr, size); } - void TearDown([[maybe_unused]] ::benchmark::State &state) override {} - size_t nextSize() override { return size; }; - static std::vector argsName() { return {"size"}; } + + static std::string name() { return Provider::name(); } private: - size_t size; + Provider provider; }; -class uniform_alloc_size : public alloc_size_interface { - using distribution = std::uniform_int_distribution; - +// TODO: assert Pool to be a pool_interface. +template class pool_allocator : public allocator_interface { public: - unsigned SetUp(::benchmark::State &state, unsigned argPos) override { - auto min = state.range(argPos++); - auto max = state.range(argPos++); - auto gran = state.range(argPos++); - if (min % gran != 0 && max % gran != 0) { - state.SkipWithError("min and max must be divisible by granularity"); - return argPos; - } + unsigned SetUp(::benchmark::State &state, unsigned r) override { + pool.SetUp(state); + return r; + } - dist.param(distribution::param_type(min / gran, max / gran)); - multiplier = gran; - return argPos; + void TearDown(::benchmark::State &state) override { pool.TearDown(state); } + + virtual void *benchAlloc(size_t size) override { + return umfPoolMalloc(pool.pool, size); } - void TearDown([[maybe_unused]] ::benchmark::State &state) override {} - size_t nextSize() override { return dist(generator) * multiplier; } - static std::vector argsName() { - return {"min size", "max size", "granularity"}; + + virtual void benchFree(void *ptr, [[maybe_unused]] size_t size) override { + umfPoolFree(pool.pool, ptr); } + static std::string name() { return Pool::name(); } + private: - std::default_random_engine generator; - distribution dist; - size_t multiplier; + Pool pool; +}; + +template +struct benchmark_interface : public benchmark::Fixture { + void SetUp(::benchmark::State &state) { + int argPos = alloc_size.SetUp(state, 0); + allocator.SetUp(state, argPos); + } + + void TearDown(::benchmark::State &state) { + alloc_size.TearDown(state); + allocator.TearDown(state); + } + + virtual void bench(::benchmark::State &state) = 0; + + static std::vector argsName() { + auto s = Size::argsName(); + auto a = Allocator::argsName(); + std::vector res = {}; + res.insert(res.end(), s.begin(), s.end()); + res.insert(res.end(), a.begin(), a.end()); + return res; + } + + virtual std::string name() { return Allocator::name(); } + virtual int64_t iterations() { return 10000; } + static void defaultArgs(Benchmark *benchmark) { + auto *bench = + static_cast *>(benchmark); + benchmark->ArgNames(bench->argsName()) + ->Name(bench->name()) + ->Iterations(bench->iterations()); + } + Size alloc_size; + Allocator allocator; }; // This class benchmarks speed of alloc() operations. @@ -231,13 +266,16 @@ class alloc_benchmark : public benchmark_interface { state.ResumeTiming(); } } - static std::vector argsName() { + + virtual std::vector argsName() { auto n = benchmark_interface::argsName(); std::vector res = {"max_allocs", "pre_allocs"}; res.insert(res.end(), n.begin(), n.end()); return res; } - static std::string name() { return base::name() + "/alloc"; } + + virtual std::string name() { return base::name() + "/alloc"; } + virtual int64_t iterations() { return 200000; } protected: using base = benchmark_interface; @@ -315,68 +353,19 @@ class multiple_malloc_free_benchmark : public alloc_benchmark { } } - static std::string name() { + virtual std::string name() { return base::base::name() + "/multiple_malloc_free"; } - static std::vector argsName() { + + virtual std::vector argsName() { auto n = benchmark_interface::argsName(); std::vector res = {"max_allocs"}; res.insert(res.end(), n.begin(), n.end()); return res; } - std::default_random_engine generator; - distribution dist; -}; - -template ::value>> -class provider_allocator : public allocator_interface { - public: - unsigned SetUp(::benchmark::State &state, unsigned r) override { - provider.SetUp(state); - return r; - } - void TearDown(::benchmark::State &state) override { - provider.TearDown(state); - } + virtual int64_t iterations() { return 2000; } - void *benchAlloc(size_t size) override { - void *ptr; - if (umfMemoryProviderAlloc(provider.provider, size, 0, &ptr) != - UMF_RESULT_SUCCESS) { - return NULL; - } - return ptr; - } - void benchFree(void *ptr, size_t size) override { - umfMemoryProviderFree(provider.provider, ptr, size); - } - static std::string name() { return Provider::name(); } - - private: - Provider provider; -}; - -// TODO: assert Pool to be a pool_interface. -template class pool_allocator : public allocator_interface { - public: - unsigned SetUp(::benchmark::State &state, unsigned r) override { - pool.SetUp(state); - return r; - } - - void TearDown(::benchmark::State &state) override { pool.TearDown(state); } - - virtual void *benchAlloc(size_t size) override { - return umfPoolMalloc(pool.pool, size); - } - virtual void benchFree(void *ptr, [[maybe_unused]] size_t size) override { - umfPoolFree(pool.pool, ptr); - } - - static std::string name() { return Pool::name(); } - - private: - Pool pool; + std::default_random_engine generator; + distribution dist; }; diff --git a/benchmark/benchmark_interfaces.hpp b/benchmark/benchmark_interfaces.hpp deleted file mode 100644 index 8681160626..0000000000 --- a/benchmark/benchmark_interfaces.hpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (C) 2024 Intel Corporation - * - * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - * - */ - -#include -#include -#include - -#include -#include -#include - -class alloc_size_interface { - public: - virtual unsigned SetUp([[maybe_unused]] ::benchmark::State &state, - [[maybe_unused]] unsigned argPos) = 0; - virtual void TearDown([[maybe_unused]] ::benchmark::State &state) = 0; - virtual size_t nextSize() = 0; - static std::vector argsName() { return {""}; }; -}; - -class allocator_interface { - public: - virtual unsigned SetUp([[maybe_unused]] ::benchmark::State &state, - [[maybe_unused]] unsigned argPos) = 0; - virtual void TearDown([[maybe_unused]] ::benchmark::State &state) = 0; - virtual void *benchAlloc(size_t size) = 0; - virtual void benchFree(void *ptr, [[maybe_unused]] size_t size) = 0; - static std::vector argsName() { return {}; } -}; - -template -struct benchmark_interface : public benchmark::Fixture { - void SetUp(::benchmark::State &state) { - int argPos = alloc_size.SetUp(state, 0); - allocator.SetUp(state, argPos); - } - void TearDown(::benchmark::State &state) { - alloc_size.TearDown(state); - allocator.TearDown(state); - } - - virtual void bench(::benchmark::State &state) = 0; - - static std::vector argsName() { - auto s = Size::argsName(); - auto a = Allocator::argsName(); - std::vector res = {}; - res.insert(res.end(), s.begin(), s.end()); - res.insert(res.end(), a.begin(), a.end()); - return res; - } - static std::string name() { return Allocator::name(); } - - Size alloc_size; - Allocator allocator; -}; - -struct provider_interface { - umf_memory_provider_handle_t provider = NULL; - virtual void SetUp(::benchmark::State &state) { - if (state.thread_index() != 0) { - return; - } - auto umf_result = - umfMemoryProviderCreate(getOps(), getParams(), &provider); - if (umf_result != UMF_RESULT_SUCCESS) { - state.SkipWithError("umfMemoryProviderCreate() failed"); - } - } - - virtual void TearDown([[maybe_unused]] ::benchmark::State &state) { - if (state.thread_index() != 0) { - return; - } - - if (provider) { - umfMemoryProviderDestroy(provider); - } - } - - virtual umf_memory_provider_ops_t *getOps() { return nullptr; } - virtual void *getParams() { return nullptr; } -}; - -template ::value>> -struct pool_interface { - virtual void SetUp(::benchmark::State &state) { - provider.SetUp(state); - if (state.thread_index() != 0) { - return; - } - auto umf_result = umfPoolCreate(getOps(state), provider.provider, - getParams(state), 0, &pool); - if (umf_result != UMF_RESULT_SUCCESS) { - state.SkipWithError("umfPoolCreate() failed"); - } - } - virtual void TearDown([[maybe_unused]] ::benchmark::State &state) { - if (state.thread_index() != 0) { - return; - } - // TODO: The scalable pool destruction process can race with other threads - // performing TLS (Thread-Local Storage) destruction. - // As a temporary workaround, we introduce a delay (sleep) - // to ensure the pool is destroyed only after all threads have completed. - // Issue: #933 - std::this_thread::sleep_for(std::chrono::milliseconds(500)); - if (pool) { - umfPoolDestroy(pool); - } - }; - - virtual umf_memory_pool_ops_t * - getOps([[maybe_unused]] ::benchmark::State &state) { - return nullptr; - } - virtual void *getParams([[maybe_unused]] ::benchmark::State &state) { - return nullptr; - } - T provider; - umf_memory_pool_handle_t pool; -}; diff --git a/benchmark/benchmark_size.hpp b/benchmark/benchmark_size.hpp new file mode 100644 index 0000000000..d17a6b2869 --- /dev/null +++ b/benchmark/benchmark_size.hpp @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#include +#include +#include + +class alloc_size_interface { + public: + virtual unsigned SetUp([[maybe_unused]] ::benchmark::State &state, + [[maybe_unused]] unsigned argPos) = 0; + virtual void TearDown([[maybe_unused]] ::benchmark::State &state) = 0; + virtual size_t nextSize() = 0; + static std::vector argsName() { return {""}; }; +}; + +class fixed_alloc_size : public alloc_size_interface { + public: + unsigned SetUp(::benchmark::State &state, unsigned argPos) override { + size = state.range(argPos); + return argPos + 1; + } + void TearDown([[maybe_unused]] ::benchmark::State &state) override {} + size_t nextSize() override { return size; }; + static std::vector argsName() { return {"size"}; } + + private: + size_t size; +}; + +class uniform_alloc_size : public alloc_size_interface { + using distribution = std::uniform_int_distribution; + + public: + unsigned SetUp(::benchmark::State &state, unsigned argPos) override { + auto min = state.range(argPos++); + auto max = state.range(argPos++); + auto gran = state.range(argPos++); + if (min % gran != 0 && max % gran != 0) { + state.SkipWithError("min and max must be divisible by granularity"); + return argPos; + } + + dist.param(distribution::param_type(min / gran, max / gran)); + multiplier = gran; + return argPos; + } + void TearDown([[maybe_unused]] ::benchmark::State &state) override {} + size_t nextSize() override { return dist(generator) * multiplier; } + static std::vector argsName() { + return {"min size", "max size", "granularity"}; + } + + private: + std::default_random_engine generator; + distribution dist; + size_t multiplier; +}; diff --git a/benchmark/benchmark_umf.hpp b/benchmark/benchmark_umf.hpp new file mode 100644 index 0000000000..389c224ed1 --- /dev/null +++ b/benchmark/benchmark_umf.hpp @@ -0,0 +1,252 @@ +/* + * Copyright (C) 2024-2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ +#include +#include + +#include +#include +#include + +#include +#include +#ifdef UMF_POOL_SCALABLE_ENABLED +#include +#endif +#include + +#ifdef UMF_POOL_DISJOINT_ENABLED +#include +#endif + +#ifdef UMF_POOL_JEMALLOC_ENABLED +#include +#endif + +struct provider_interface { + using params_ptr = std::unique_ptr; + + umf_memory_provider_handle_t provider = NULL; + virtual void SetUp(::benchmark::State &state) { + if (state.thread_index() != 0) { + return; + } + auto params = getParams(state); + auto umf_result = + umfMemoryProviderCreate(getOps(state), params.get(), &provider); + if (umf_result != UMF_RESULT_SUCCESS) { + state.SkipWithError("umfMemoryProviderCreate() failed"); + } + } + + virtual void TearDown([[maybe_unused]] ::benchmark::State &state) { + if (state.thread_index() != 0) { + return; + } + + if (provider) { + umfMemoryProviderDestroy(provider); + } + } + + virtual umf_memory_provider_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) { + return nullptr; + } + + virtual params_ptr getParams([[maybe_unused]] ::benchmark::State &state) { + return {nullptr, [](void *) {}}; + } +}; + +template ::value>> +struct pool_interface { + using params_ptr = std::unique_ptr; + + virtual void SetUp(::benchmark::State &state) { + provider.SetUp(state); + if (state.thread_index() != 0) { + return; + } + auto params = getParams(state); + auto umf_result = umfPoolCreate(getOps(state), provider.provider, + params.get(), 0, &pool); + if (umf_result != UMF_RESULT_SUCCESS) { + state.SkipWithError("umfPoolCreate() failed"); + } + } + virtual void TearDown([[maybe_unused]] ::benchmark::State &state) { + if (state.thread_index() != 0) { + return; + } + // TODO: The scalable pool destruction process can race with other threads + // performing TLS (Thread-Local Storage) destruction. + // As a temporary workaround, we introduce a delay (sleep) + // to ensure the pool is destroyed only after all threads have completed. + // Issue: #933 + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + if (pool) { + umfPoolDestroy(pool); + } + }; + + virtual umf_memory_pool_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) { + return nullptr; + } + virtual params_ptr getParams([[maybe_unused]] ::benchmark::State &state) { + return {nullptr, [](void *) {}}; + } + T provider; + umf_memory_pool_handle_t pool; +}; + +class allocator_interface { + public: + virtual unsigned SetUp([[maybe_unused]] ::benchmark::State &state, + [[maybe_unused]] unsigned argPos) = 0; + virtual void TearDown([[maybe_unused]] ::benchmark::State &state) = 0; + virtual void *benchAlloc(size_t size) = 0; + virtual void benchFree(void *ptr, [[maybe_unused]] size_t size) = 0; + static std::vector argsName() { return {}; } +}; + +struct glibc_malloc : public allocator_interface { + unsigned SetUp([[maybe_unused]] ::benchmark::State &state, + unsigned argPos) override { + return argPos; + } + void TearDown([[maybe_unused]] ::benchmark::State &state) override{}; + void *benchAlloc(size_t size) override { return malloc(size); } + void benchFree(void *ptr, [[maybe_unused]] size_t size) override { + free(ptr); + } + static std::string name() { return "glibc"; } +}; + +struct os_provider : public provider_interface { + provider_interface::params_ptr + getParams(::benchmark::State &state) override { + umf_os_memory_provider_params_handle_t raw_params = nullptr; + umfOsMemoryProviderParamsCreate(&raw_params); + if (!raw_params) { + state.SkipWithError("Failed to create os provider params"); + return {nullptr, [](void *) {}}; + } + + // Use a lambda as the custom deleter + auto deleter = [](void *p) { + auto handle = + static_cast(p); + umfOsMemoryProviderParamsDestroy(handle); + }; + + return {static_cast(raw_params), deleter}; + } + + umf_memory_provider_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) override { + return umfOsMemoryProviderOps(); + } + static std::string name() { return "os_provider"; } +}; + +template +struct proxy_pool : public pool_interface { + umf_memory_pool_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) override { + return umfProxyPoolOps(); + } + + static std::string name() { return "proxy_pool<" + Provider::name() + ">"; } +}; + +#ifdef UMF_POOL_DISJOINT_ENABLED +template +struct disjoint_pool : public pool_interface { + umf_memory_pool_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) override { + return umfDisjointPoolOps(); + } + + typename pool_interface::params_ptr + getParams(::benchmark::State &state) override { + umf_disjoint_pool_params_handle_t raw_params = nullptr; + auto ret = umfDisjointPoolParamsCreate(&raw_params); + if (ret != UMF_RESULT_SUCCESS) { + state.SkipWithError("Failed to create disjoint pool params"); + return {nullptr, [](void *) {}}; + } + + typename pool_interface::params_ptr params( + raw_params, [](void *p) { + umfDisjointPoolParamsDestroy( + static_cast(p)); + }); + + ret = umfDisjointPoolParamsSetSlabMinSize(raw_params, 4096); + if (ret != UMF_RESULT_SUCCESS) { + state.SkipWithError("Failed to set slab min size"); + return {nullptr, [](void *) {}}; + } + + ret = umfDisjointPoolParamsSetCapacity(raw_params, 4); + if (ret != UMF_RESULT_SUCCESS) { + state.SkipWithError("Failed to set capacity"); + return {nullptr, [](void *) {}}; + } + + ret = umfDisjointPoolParamsSetMinBucketSize(raw_params, 4096); + if (ret != UMF_RESULT_SUCCESS) { + state.SkipWithError("Failed to set min bucket size"); + return {nullptr, [](void *) {}}; + } + + ret = umfDisjointPoolParamsSetMaxPoolableSize(raw_params, 4096 * 16); + if (ret != UMF_RESULT_SUCCESS) { + state.SkipWithError("Failed to set max poolable size"); + return {nullptr, [](void *) {}}; + } + + return params; + } + + static std::string name() { + return "disjoint_pool<" + Provider::name() + ">"; + } +}; +#endif + +#ifdef UMF_POOL_JEMALLOC_ENABLED +template +struct jemalloc_pool : public pool_interface { + umf_memory_pool_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) override { + return umfJemallocPoolOps(); + } + + static std::string name() { + return "jemalloc_pool<" + Provider::name() + ">"; + } +}; +#endif + +#ifdef UMF_POOL_SCALABLE_ENABLED +template +struct scalable_pool : public pool_interface { + virtual umf_memory_pool_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) override { + return umfScalablePoolOps(); + } + + static std::string name() { + return "scalable_pool<" + Provider::name() + ">"; + } +}; +#endif diff --git a/benchmark/multithread.cpp b/benchmark/multithread.cpp index 4558942ecb..ecc2385292 100644 --- a/benchmark/multithread.cpp +++ b/benchmark/multithread.cpp @@ -139,11 +139,13 @@ int main() { // ctest looks for "PASSED" in the output std::cout << "PASSED" << std::endl; +#if defined(UMF_POOL_DISJOINT_ENABLED) ret = umfDisjointPoolParamsDestroy(hDisjointParams); if (ret != UMF_RESULT_SUCCESS) { std::cerr << "disjoint pool params destroy failed" << std::endl; return -1; } +#endif return 0; } diff --git a/benchmark/ubench.c b/benchmark/ubench.c index 5f1bfe9e48..845dc881db 100644 --- a/benchmark/ubench.c +++ b/benchmark/ubench.c @@ -445,8 +445,8 @@ static void do_ipc_get_put_benchmark(alloc_t *allocs, size_t num_allocs, } } -int create_level_zero_params(ze_context_handle_t *context, - ze_device_handle_t *device) { +static int create_level_zero_params(ze_context_handle_t *context, + ze_device_handle_t *device) { uint32_t driver_idx = 0; ze_driver_handle_t driver = NULL; diff --git a/cmake/helpers.cmake b/cmake/helpers.cmake index 2544a15186..2d14e2f45f 100644 --- a/cmake/helpers.cmake +++ b/cmake/helpers.cmake @@ -1,4 +1,4 @@ -# Copyright (C) 2023-2024 Intel Corporation +# Copyright (C) 2023-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -232,9 +232,9 @@ function(add_umf_target_compile_options name) PRIVATE -fPIC -Wall -Wextra - -Wpedantic -Wformat-security - -Wcast-qual + -Wno-cast-qual # TODO: remove this when const qualifier drop + # will be solved in CTL $<$:-fdiagnostics-color=auto>) if(CMAKE_BUILD_TYPE STREQUAL "Release") target_compile_definitions(${name} PRIVATE -D_FORTIFY_SOURCE=2) @@ -378,6 +378,9 @@ function(add_umf_library) elseif(LINUX) target_link_options(${ARG_NAME} PRIVATE "-Wl,--version-script=${ARG_LINUX_MAP_FILE}") + if(CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") + target_link_options(${ARG_NAME} PRIVATE -no-intel-lib) + endif() endif() endif() @@ -387,7 +390,8 @@ function(add_umf_library) ${ARG_NAME} PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include ${UMF_CMAKE_SOURCE_DIR}/src/utils - ${UMF_CMAKE_SOURCE_DIR}/src/base_alloc) + ${UMF_CMAKE_SOURCE_DIR}/src/base_alloc + ${UMF_CMAKE_SOURCE_DIR}/src/coarse) add_umf_target_compile_options(${ARG_NAME}) add_umf_target_link_options(${ARG_NAME}) endfunction() diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000..737bb12595 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,32 @@ +# Documentation + +To generate HTML documentation, run the `generate_docs.py` script from any sub-directory of the repository (most likely `build`). +To display the proper version of UMF in the documentation title, set the `UMF_VERSION` variable before running the script. + +```bash +cd build +$ UMF_VERSION= python ../docs/generate_docs.py +``` + +Documentation can also be built using the build target 'docs' (see details below). + +This script will create `./docs_build` sub-directory, where the intermediate and final files +will be created. HTML docs will be in the `./docs_build/generated/html` directory. + +## make docs + +To run documentation generation via build target use CMake commands below. +To enable this target, python executable (in required version) has to be found in the system. + +```bash +cmake -B build +cmake --build build --target docs +``` + +## Requirements + +Script to generate HTML docs requires: + +* [Doxygen](http://www.doxygen.nl/) at least v1.9.1 +* [Python](https://www.python.org/downloads/) at least v3.8 +* and python pip requirements, as defined in `third_party/requirements.txt` diff --git a/scripts/assets/images/intro_architecture.png b/docs/assets/images/intro_architecture.png similarity index 100% rename from scripts/assets/images/intro_architecture.png rename to docs/assets/images/intro_architecture.png diff --git a/scripts/docs_config/Doxyfile b/docs/config/Doxyfile similarity index 99% rename from scripts/docs_config/Doxyfile rename to docs/config/Doxyfile index 43ff2a6037..6309463745 100644 --- a/scripts/docs_config/Doxyfile +++ b/docs/config/Doxyfile @@ -445,7 +445,7 @@ INLINE_SIMPLE_STRUCTS = NO # types are typedef'ed and only the typedef is referenced, never the tag name. # The default value is: NO. -TYPEDEF_HIDES_STRUCT = NO +TYPEDEF_HIDES_STRUCT = YES # The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This # cache is used to resolve symbols given their name and scope. Since this can be @@ -2058,7 +2058,7 @@ GENERATE_XML = YES # The default directory is: xml. # This tag requires that the tag GENERATE_XML is set to YES. -XML_OUTPUT = ../docs/xml +XML_OUTPUT = docs_build/doxyxml # If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program # listings (including syntax highlighting and cross-referencing information) to diff --git a/scripts/docs_config/api.rst b/docs/config/api.rst similarity index 91% rename from scripts/docs_config/api.rst rename to docs/config/api.rst index 7f734cad24..1c20d709c2 100644 --- a/scripts/docs_config/api.rst +++ b/docs/config/api.rst @@ -58,6 +58,9 @@ supported by the Proxy Pool. Scalable Pool ------------------------------------------ + +A oneTBB-based memory pool manager. + .. doxygenfile:: pool_scalable.h :sections: define enum typedef func var @@ -80,17 +83,12 @@ and operate on the provider. .. doxygenfile:: memory_provider.h :sections: define enum typedef func var -Coarse Provider +Fixed Memory Provider ------------------------------------------ -A memory provider that can provide memory from: +A memory provider that can provide memory from a given preallocated buffer. -1) A given pre-allocated buffer (the fixed-size memory provider option) or -2) From an additional upstream provider (e.g. provider that does not support - the free() operation like the File memory provider or the DevDax memory - provider - see below). - -.. doxygenfile:: provider_coarse.h +.. doxygenfile:: provider_fixed_memory.h :sections: define enum typedef func var OS Memory Provider @@ -109,10 +107,18 @@ A memory provider that provides memory from L0 device. .. doxygenfile:: provider_level_zero.h :sections: define enum typedef func var +CUDA Provider +------------------------------------------ + +A memory provider that provides memory from CUDA device. + +.. doxygenfile:: provider_cuda.h + :sections: define enum typedef func var + DevDax Memory Provider ------------------------------------------ -A memory provider that provides memory from a device DAX (a character device file /dev/daxX.Y). +A memory provider that provides memory from a device DAX (a character device file like /dev/daxX.Y). .. doxygenfile:: provider_devdax_memory.h :sections: define enum typedef func var diff --git a/scripts/docs_config/conf.py b/docs/config/conf.py similarity index 79% rename from scripts/docs_config/conf.py rename to docs/config/conf.py index 77d9856274..ae698ba98c 100644 --- a/scripts/docs_config/conf.py +++ b/docs/config/conf.py @@ -1,3 +1,5 @@ +import os + # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full @@ -22,15 +24,22 @@ author = "Intel" # The full version, including alpha/beta/rc tags -release = "0.10.1" - +release = os.getenv("UMF_VERSION", "") +print( + f"UMF_VERSION used in docs: {release}" + if release != "" + else "please set UMF_VERSION environment variable before running this script" +) # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["breathe"] +extensions = ["breathe", "sphinxcontrib.spelling"] + +spelling_show_suggestions = True +spelling_word_list_filename = "spelling_exceptions.txt" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -49,7 +58,9 @@ # -- Extension configuration ------------------------------------------------- # -- Options for breathe extension ------------------------------------------- -breathe_projects = {project: "../../docs/xml"} +# 'doxyxml' dir is generated with Doxygen; it's supposed to be in a directory +# one above the config directory. +breathe_projects = {project: "../doxyxml"} breathe_default_project = project breathe_show_include = False breathe_default_members = ("members", "undoc-members") diff --git a/scripts/docs_config/examples.rst b/docs/config/examples.rst similarity index 99% rename from scripts/docs_config/examples.rst rename to docs/config/examples.rst index c58e7fc223..4eeea6aa95 100644 --- a/scripts/docs_config/examples.rst +++ b/docs/config/examples.rst @@ -178,7 +178,7 @@ by a different library and the caller of the :any:`umfGetIPCHandle` function may The :any:`umfGetIPCHandle` function returns the IPC handle and its size. The IPC handle is a byte-copyable opaque data structure. The :any:`umf_ipc_handle_t` type is defined as a pointer to a byte array. The size of the handle might be different for different memory provider types. The code snippet below demonstrates how the IPC handle can -be serialized for marshalling purposes. +be serialized for marshaling purposes. .. code-block:: c diff --git a/scripts/docs_config/glossary.rst b/docs/config/glossary.rst similarity index 100% rename from scripts/docs_config/glossary.rst rename to docs/config/glossary.rst diff --git a/scripts/docs_config/index.rst b/docs/config/index.rst similarity index 100% rename from scripts/docs_config/index.rst rename to docs/config/index.rst diff --git a/scripts/docs_config/introduction.rst b/docs/config/introduction.rst similarity index 100% rename from scripts/docs_config/introduction.rst rename to docs/config/introduction.rst diff --git a/docs/config/spelling_exceptions.txt b/docs/config/spelling_exceptions.txt new file mode 100644 index 0000000000..d4e40a3ec8 --- /dev/null +++ b/docs/config/spelling_exceptions.txt @@ -0,0 +1,74 @@ +addr +allocatable +allocator +allocators +calloc +CXL +copyable +customizable +daxX +deallocation +deallocating +deallocations +Devdax +dev +Globals +hMemtarget +hPool +hProvider +highPtr +io +interprocess +ipc +jemalloc +lowPtr +malloc +maxnode +mem +mempolicies +mempolicy +Mempolicy +memspace +Memspace +memspaces +Memtarget +memtarget +memtargets +middleware +multithreading +Nodemask +nodemask +numa +oneAPI +oneTBB +os +params +partList +pid +poolable +preallocated +providerIpcData +providential +ptr +realloc +Scalable +scalable +stdout +Tiering +tiering +topologies +umf +umfGetIPCHandle +umfMemoryProviderAlloc +umfMemoryProviderGetLastNativeError +umfMemoryProviderOpenIPCHandle +umfOsMemoryProviderParamsDestroy +umfPool +umfPoolCalloc +umfPoolDestroy +umfPoolGetTag +umfPoolMallocUsableSize +umfPoolRealloc +umfMemspaceUserFilter +umfMemspaceMemtargetAdd +unfreed \ No newline at end of file diff --git a/scripts/generate_docs.py b/docs/generate_docs.py similarity index 71% rename from scripts/generate_docs.py rename to docs/generate_docs.py index d5b2a01282..1697eacfe6 100644 --- a/scripts/generate_docs.py +++ b/docs/generate_docs.py @@ -6,17 +6,20 @@ """ from pathlib import Path -from shutil import rmtree +from shutil import rmtree, copytree import subprocess # nosec B404 import time def _check_cwd() -> None: - script_path = Path(__file__).resolve().parent cwd = Path.cwd() - if script_path != cwd: + include_dir = Path(cwd, "../include") + # Verify if include dir is one level up (as defined in Doxyfile) + if not include_dir.exists(): print( - f"{__file__} script has to be run from the 'scripts' directory. Terminating..." + f"Include directory {include_dir.resolve()} not found! " + "Please run this script from /build!", + flush=True, ) exit(1) @@ -66,8 +69,17 @@ def _generate_html(config_path: Path, docs_path: Path) -> None: def main() -> None: _check_cwd() - config_path = Path("docs_config").resolve() - docs_path = Path("..", "docs").resolve() + + script_dir = Path(__file__).resolve().parent + docs_build_path = Path("docs_build").resolve() + + # Sphinx and breathe require access to a Doxygen generated dir ('doxyxml') + # so we copy the whole content of the 'docs' dir to the build dir. + copytree(Path(script_dir), docs_build_path, dirs_exist_ok=True) + + config_path = Path(docs_build_path, "config").resolve() + docs_path = Path(docs_build_path, "generated").resolve() + start = time.time() _prepare_docs_dir(docs_path) _generate_xml(config_path, docs_path) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 942579a303..986ad56412 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -282,10 +282,13 @@ if(LINUX) add_umf_executable( NAME ${EXAMPLE_NAME} SRCS dram_and_fsdax/dram_and_fsdax.c - LIBS umf jemalloc_pool) + LIBS umf) - target_link_directories(${EXAMPLE_NAME} PRIVATE - ${LIBHWLOC_LIBRARY_DIRS}) + target_link_options(${EXAMPLE_NAME} PRIVATE "-Wl,--no-as-needed,-ldl") + + target_link_directories( + ${EXAMPLE_NAME} PRIVATE ${LIBHWLOC_LIBRARY_DIRS} + ${JEMALLOC_LIBRARY_DIRS}) add_test( NAME ${EXAMPLE_NAME} diff --git a/examples/cmake/FindJEMALLOC.cmake b/examples/cmake/FindJEMALLOC.cmake index 89d488ecc0..e6db190d4a 100644 --- a/examples/cmake/FindJEMALLOC.cmake +++ b/examples/cmake/FindJEMALLOC.cmake @@ -2,9 +2,11 @@ # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -message(STATUS "Checking for module 'jemalloc' using find_library()") +message( + STATUS "Looking for the static 'libjemalloc.a' library using find_library()" +) -find_library(JEMALLOC_LIBRARY NAMES libjemalloc jemalloc) +find_library(JEMALLOC_LIBRARY NAMES libjemalloc.a jemalloc.a) set(JEMALLOC_LIBRARIES ${JEMALLOC_LIBRARY}) get_filename_component(JEMALLOC_LIB_DIR ${JEMALLOC_LIBRARIES} DIRECTORY) diff --git a/examples/custom_file_provider/custom_file_provider.c b/examples/custom_file_provider/custom_file_provider.c index ffa61d63f5..b17fdc0f08 100644 --- a/examples/custom_file_provider/custom_file_provider.c +++ b/examples/custom_file_provider/custom_file_provider.c @@ -62,7 +62,8 @@ static umf_result_t file_init(void *params, void **provider) { // Open the file file_provider->fd = open(file_params->filename, O_RDWR | O_CREAT, 0666); if (file_provider->fd < 0) { - perror("Failed to open file"); + perror("open()"); + fprintf(stderr, "Failed to open the file: %s\n", file_params->filename); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto cleanup_malloc; } @@ -237,11 +238,11 @@ static umf_memory_provider_ops_t file_ops = { .initialize = file_init, .finalize = file_deinit, .alloc = file_alloc, + .free = file_free, .get_name = file_get_name, .get_last_native_error = file_get_last_native_error, .get_recommended_page_size = file_get_recommended_page_size, .get_min_page_size = file_get_min_page_size, - .ext.free = file_free, }; // Main function diff --git a/examples/dram_and_fsdax/CMakeLists.txt b/examples/dram_and_fsdax/CMakeLists.txt index 0d0bf25935..dcb538085e 100644 --- a/examples/dram_and_fsdax/CMakeLists.txt +++ b/examples/dram_and_fsdax/CMakeLists.txt @@ -21,26 +21,15 @@ if(NOT LIBHWLOC_FOUND) find_package(LIBHWLOC 2.3.0 REQUIRED hwloc) endif() -pkg_check_modules(JEMALLOC jemalloc) -if(NOT JEMALLOC_FOUND) - find_package(JEMALLOC REQUIRED jemalloc) -endif() - # build the example set(EXAMPLE_NAME umf_example_dram_and_fsdax) add_executable(${EXAMPLE_NAME} dram_and_fsdax.c) target_include_directories(${EXAMPLE_NAME} PRIVATE ${LIBUMF_INCLUDE_DIRS}) -target_link_directories( - ${EXAMPLE_NAME} - PRIVATE - ${LIBUMF_LIBRARY_DIRS} - ${LIBHWLOC_LIBRARY_DIRS} - ${JEMALLOC_LIBRARY_DIRS}) +target_link_directories(${EXAMPLE_NAME} PRIVATE ${LIBUMF_LIBRARY_DIRS} + ${LIBHWLOC_LIBRARY_DIRS}) -target_link_libraries( - ${EXAMPLE_NAME} PRIVATE hwloc jemalloc_pool ${JEMALLOC_LIBRARIES} - ${LIBUMF_LIBRARIES}) +target_link_libraries(${EXAMPLE_NAME} PRIVATE hwloc ${LIBUMF_LIBRARIES}) # an optional part - adds a test of this example add_test( @@ -56,6 +45,6 @@ if(LINUX) TEST ${EXAMPLE_NAME} PROPERTY ENVIRONMENT_MODIFICATION - "LD_LIBRARY_PATH=path_list_append:${LIBUMF_LIBRARY_DIRS};LD_LIBRARY_PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS};LD_LIBRARY_PATH=path_list_append:${JEMALLOC_LIBRARY_DIRS}" + "LD_LIBRARY_PATH=path_list_append:${LIBUMF_LIBRARY_DIRS};LD_LIBRARY_PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS}" ) endif() diff --git a/examples/dram_and_fsdax/dram_and_fsdax.c b/examples/dram_and_fsdax/dram_and_fsdax.c index 26f4517281..970242e109 100644 --- a/examples/dram_and_fsdax/dram_and_fsdax.c +++ b/examples/dram_and_fsdax/dram_and_fsdax.c @@ -78,41 +78,14 @@ static umf_memory_pool_handle_t create_fsdax_pool(const char *path) { } // Create an FSDAX memory pool - // - // The file memory provider does not support the free operation - // (`umfMemoryProviderFree()` always returns `UMF_RESULT_ERROR_NOT_SUPPORTED`), - // so it should be used with a pool manager that will take over - // the managing of the provided memory - for example the jemalloc pool - // with the `disable_provider_free` parameter set to true. - umf_jemalloc_pool_params_handle_t pool_params; - umf_result = umfJemallocPoolParamsCreate(&pool_params); - if (umf_result != UMF_RESULT_SUCCESS) { - fprintf(stderr, "Failed to create jemalloc params!\n"); - umfMemoryProviderDestroy(provider_fsdax); - return NULL; - } - umf_result = umfJemallocPoolParamsSetKeepAllMemory(pool_params, true); - if (umf_result != UMF_RESULT_SUCCESS) { - fprintf(stderr, "Failed to set KeepAllMemory!\n"); - umfMemoryProviderDestroy(provider_fsdax); - return NULL; - } - - // Create an FSDAX memory pool - umf_result = - umfPoolCreate(umfJemallocPoolOps(), provider_fsdax, pool_params, - UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &pool_fsdax); + umf_result = umfPoolCreate(umfJemallocPoolOps(), provider_fsdax, NULL, + UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &pool_fsdax); if (umf_result != UMF_RESULT_SUCCESS) { fprintf(stderr, "Failed to create an FSDAX memory pool!\n"); umfMemoryProviderDestroy(provider_fsdax); return NULL; } - umf_result = umfJemallocPoolParamsDestroy(pool_params); - if (umf_result != UMF_RESULT_SUCCESS) { - fprintf(stderr, "Failed to destroy jemalloc params!\n"); - } - return pool_fsdax; } diff --git a/examples/ipc_ipcapi/ipc_ipcapi_anon_fd.sh b/examples/ipc_ipcapi/ipc_ipcapi_anon_fd.sh index 615271eebb..2eb9409daf 100755 --- a/examples/ipc_ipcapi/ipc_ipcapi_anon_fd.sh +++ b/examples/ipc_ipcapi/ipc_ipcapi_anon_fd.sh @@ -1,5 +1,5 @@ # -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2024-2025 Intel Corporation # # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -16,16 +16,8 @@ PORT=$(( 1024 + ( $$ % ( 65535 - 1024 )))) # to obtain a duplicate of another process's file descriptor. # Permission to duplicate another process's file descriptor # is governed by a ptrace access mode PTRACE_MODE_ATTACH_REALCREDS check (see ptrace(2)) -# that can be changed using the /proc/sys/kernel/yama/ptrace_scope interface. -PTRACE_SCOPE_FILE="/proc/sys/kernel/yama/ptrace_scope" -VAL=0 -if [ -f $PTRACE_SCOPE_FILE ]; then - PTRACE_SCOPE_VAL=$(cat $PTRACE_SCOPE_FILE) - if [ $PTRACE_SCOPE_VAL -ne $VAL ]; then - echo "SKIP: ptrace_scope is not set to 0 (classic ptrace permissions) - skipping the test" - exit 125 # skip code defined in CMakeLists.txt - fi -fi +# In the producer binary used in this example prctl(PR_SET_PTRACER, getppid()) is used +# to allow consumer to duplicate file descriptor of producer. UMF_LOG_VAL="level:debug;flush:debug;output:stderr;pid:yes" diff --git a/examples/ipc_ipcapi/ipc_ipcapi_producer.c b/examples/ipc_ipcapi/ipc_ipcapi_producer.c index 4157e8284f..9082302ac9 100644 --- a/examples/ipc_ipcapi/ipc_ipcapi_producer.c +++ b/examples/ipc_ipcapi/ipc_ipcapi_producer.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -69,6 +70,21 @@ int main(int argc, char *argv[]) { int port = atoi(argv[1]); + // The prctl() function with PR_SET_PTRACER is used here to allow parent process and its children + // to ptrace the current process. This is necessary because UMF's memory providers on Linux (except CUDA) + // use the pidfd_getfd(2) system call to duplicate another process's file descriptor, which is + // governed by ptrace permissions. By default on Ubuntu /proc/sys/kernel/yama/ptrace_scope is + // set to 1 ("restricted ptrace"), which prevents pidfd_getfd from working unless ptrace_scope + // is set to 0. + // To overcome this limitation without requiring users to change the ptrace_scope + // setting (which requires root privileges), we use prctl() to allow the consumer process + // to copy producer's file descriptor, even when ptrace_scope is set to 1. + ret = prctl(PR_SET_PTRACER, getppid()); + if (ret == -1) { + perror("PR_SET_PTRACER may be not supported. prctl() call failed"); + goto err_end; + } + umf_memory_provider_handle_t OS_memory_provider = NULL; umf_os_memory_provider_params_handle_t os_params = NULL; enum umf_result_t umf_result; @@ -259,6 +275,7 @@ int main(int argc, char *argv[]) { err_destroy_OS_params: umfOsMemoryProviderParamsDestroy(os_params); +err_end: if (ret == 0) { fprintf(stderr, "[producer] Shutting down (status OK) ...\n"); } else if (ret == 1) { diff --git a/include/umf.h b/include/umf.h index 3e2d827991..57bebef8a9 100644 --- a/include/umf.h +++ b/include/umf.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/include/umf/base.h b/include/umf/base.h index 53378195d2..32d84771f3 100644 --- a/include/umf/base.h +++ b/include/umf/base.h @@ -28,7 +28,7 @@ extern "C" { #define UMF_MINOR_VERSION(_ver) (_ver & 0x0000ffff) /// @brief Current version of the UMF headers -#define UMF_VERSION_CURRENT UMF_MAKE_VERSION(0, 10) +#define UMF_VERSION_CURRENT UMF_MAKE_VERSION(0, 11) /// @brief Operation results typedef enum umf_result_t { diff --git a/include/umf/memory_pool.h b/include/umf/memory_pool.h index a93d400f92..ae5e67a969 100644 --- a/include/umf/memory_pool.h +++ b/include/umf/memory_pool.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023-2024 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -140,7 +140,7 @@ umf_result_t umfFree(void *ptr); /// * Implementations *must* store the error code in thread-local /// storage prior to returning NULL from the allocation functions. /// -/// * If the last allocation/de-allocation operation succeeded, the value returned by +/// * If the last allocation/deallocation operation succeeded, the value returned by /// this function is unspecified. /// /// * The application *may* call this function from simultaneous threads. @@ -170,6 +170,22 @@ umf_memory_pool_handle_t umfPoolByPtr(const void *ptr); umf_result_t umfPoolGetMemoryProvider(umf_memory_pool_handle_t hPool, umf_memory_provider_handle_t *hProvider); +/// +/// @brief Set a custom tag on the memory pool that can be later retrieved using umfPoolGetTag. +/// @param hPool specified memory pool +/// @param tag tag to be set +/// @param oldTag [out][optional] previous tag set on the memory pool +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfPoolSetTag(umf_memory_pool_handle_t hPool, void *tag, + void **oldTag); + +/// +/// @brief Retrieve the tag associated with the memory pool or NULL if no tag is set. +/// @param hPool specified memory pool +/// @param tag [out] tag associated with the memory pool +/// @return UMF_RESULT_SUCCESS on success. +umf_result_t umfPoolGetTag(umf_memory_pool_handle_t hPool, void **tag); + #ifdef __cplusplus } #endif diff --git a/include/umf/memory_pool_ops.h b/include/umf/memory_pool_ops.h index 67afdd1669..829f49fb76 100644 --- a/include/umf/memory_pool_ops.h +++ b/include/umf/memory_pool_ops.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/include/umf/memory_provider_ops.h b/include/umf/memory_provider_ops.h index 0b9c7cfce3..a61e0aad07 100644 --- a/include/umf/memory_provider_ops.h +++ b/include/umf/memory_provider_ops.h @@ -22,15 +22,6 @@ extern "C" { /// can keep them NULL. /// typedef struct umf_memory_provider_ext_ops_t { - /// - /// @brief Frees the memory space pointed by \p ptr from the memory \p provider - /// @param provider pointer to the memory provider - /// @param ptr pointer to the allocated memory to free - /// @param size size of the allocation - /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure - /// - umf_result_t (*free)(void *provider, void *ptr, size_t size); - /// /// @brief Discard physical pages within the virtual memory mapping associated at the given addr /// and \p size. This call is asynchronous and may delay purging the pages indefinitely. @@ -181,6 +172,15 @@ typedef struct umf_memory_provider_ops_t { umf_result_t (*alloc)(void *provider, size_t size, size_t alignment, void **ptr); + /// + /// @brief Frees the memory space pointed by \p ptr from the memory \p provider + /// @param provider pointer to the memory provider + /// @param ptr pointer to the allocated memory to free + /// @param size size of the allocation + /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure + /// + umf_result_t (*free)(void *provider, void *ptr, size_t size); + /// /// @brief Retrieve string representation of the underlying provider specific /// result reported by the last API that returned diff --git a/include/umf/memtarget.h b/include/umf/memtarget.h index d74947f14e..55ca30919a 100644 --- a/include/umf/memtarget.h +++ b/include/umf/memtarget.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/include/umf/pools/pool_disjoint.h b/include/umf/pools/pool_disjoint.h index fdf682ae5b..d268a1dac4 100644 --- a/include/umf/pools/pool_disjoint.h +++ b/include/umf/pools/pool_disjoint.h @@ -1,6 +1,11 @@ -// Copyright (C) 2023-2024 Intel Corporation -// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +/* + * + * Copyright (C) 2023-2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ #pragma once #ifdef __cplusplus @@ -87,7 +92,7 @@ umfDisjointPoolParamsSetTrace(umf_disjoint_pool_params_handle_t hParams, /// @brief Set shared limits for disjoint pool. /// @param hParams handle to the parameters of the disjoint pool. -/// @param hSharedLimits handle tp the shared limits. +/// @param hSharedLimits handle to the shared limits. /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. umf_result_t umfDisjointPoolParamsSetSharedLimits( umf_disjoint_pool_params_handle_t hParams, diff --git a/include/umf/pools/pool_jemalloc.h b/include/umf/pools/pool_jemalloc.h index 0cbecd38f7..5974e6440a 100644 --- a/include/umf/pools/pool_jemalloc.h +++ b/include/umf/pools/pool_jemalloc.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -14,35 +14,8 @@ extern "C" { #endif -#include #include -struct umf_jemalloc_pool_params_t; - -/// @brief handle to the parameters of the jemalloc pool. -typedef struct umf_jemalloc_pool_params_t *umf_jemalloc_pool_params_handle_t; - -/// @brief Create a struct to store parameters of jemalloc pool. -/// @param hParams [out] handle to the newly created parameters struct. -/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. -umf_result_t -umfJemallocPoolParamsCreate(umf_jemalloc_pool_params_handle_t *hParams); - -/// @brief Destroy parameters struct. -/// @param hParams handle to the parameters of the jemalloc pool. -/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. -umf_result_t -umfJemallocPoolParamsDestroy(umf_jemalloc_pool_params_handle_t hParams); - -/// @brief Set if \p umfMemoryProviderFree() should never be called. -/// @param hParams handle to the parameters of the jemalloc pool. -/// @param keepAllMemory \p true if the jemalloc pool should not call -/// \p umfMemoryProviderFree, \p false otherwise. -/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. -umf_result_t -umfJemallocPoolParamsSetKeepAllMemory(umf_jemalloc_pool_params_handle_t hParams, - bool keepAllMemory); - umf_memory_pool_ops_t *umfJemallocPoolOps(void); #ifdef __cplusplus diff --git a/include/umf/pools/pool_scalable.h b/include/umf/pools/pool_scalable.h index 072169b68c..1915ad0b7a 100644 --- a/include/umf/pools/pool_scalable.h +++ b/include/umf/pools/pool_scalable.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/include/umf/providers/provider_coarse.h b/include/umf/providers/provider_coarse.h deleted file mode 100644 index 6ed6e0fbc9..0000000000 --- a/include/umf/providers/provider_coarse.h +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (C) 2023-2024 Intel Corporation - * - * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -*/ - -#ifndef UMF_COARSE_PROVIDER_H -#define UMF_COARSE_PROVIDER_H - -#include -#include - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/// @brief Coarse Memory Provider allocation strategy -typedef enum coarse_memory_provider_strategy_t { - /// Always allocate a free block of the (size + alignment) size - /// and cut out the properly aligned part leaving two remaining parts. - /// It is the fastest strategy but causes memory fragmentation - /// when alignment is greater than 0. - /// It is the best strategy when alignment always equals 0. - UMF_COARSE_MEMORY_STRATEGY_FASTEST = 0, - - /// Check if the first free block of the 'size' size has the correct alignment. - /// If not, use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. - UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE, - - /// Look through all free blocks of the 'size' size - /// and choose the first one with the correct alignment. - /// If none of them had the correct alignment, - /// use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. - UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE, - - /// The maximum value (it has to be the last one). - UMF_COARSE_MEMORY_STRATEGY_MAX -} coarse_memory_provider_strategy_t; - -/// @brief Coarse Memory Provider settings struct. -typedef struct coarse_memory_provider_params_t { - /// Handle to the upstream memory provider. - /// It has to be NULL if init_buffer is set - /// (exactly one of them has to be non-NULL). - umf_memory_provider_handle_t upstream_memory_provider; - - /// Memory allocation strategy. - /// See coarse_memory_provider_strategy_t for details. - coarse_memory_provider_strategy_t allocation_strategy; - - /// A pre-allocated buffer that will be the only memory that - /// the coarse provider can provide (the fixed-size memory provider option). - /// If it is non-NULL, `init_buffer_size ` has to contain its size. - /// It has to be NULL if upstream_memory_provider is set - /// (exactly one of them has to be non-NULL). - void *init_buffer; - - /// Size of the initial buffer: - /// 1) `init_buffer` if it is non-NULL xor - /// 2) that will be allocated from the upstream_memory_provider - /// (if it is non-NULL) in the `.initialize` operation. - size_t init_buffer_size; - - /// When it is true and the upstream_memory_provider is given, - /// the init buffer (of `init_buffer_size` bytes) would be pre-allocated - /// during creation time using the `upstream_memory_provider`. - /// If upstream_memory_provider is not given, - /// the init_buffer is always used instead - /// (regardless of the value of this parameter). - bool immediate_init_from_upstream; - - /// Destroy upstream_memory_provider in finalize(). - bool destroy_upstream_memory_provider; -} coarse_memory_provider_params_t; - -/// @brief Coarse Memory Provider stats (TODO move to CTL) -typedef struct coarse_memory_provider_stats_t { - /// Total allocation size. - size_t alloc_size; - - /// Size of used memory. - size_t used_size; - - /// Number of memory blocks allocated from the upstream provider. - size_t num_upstream_blocks; - - /// Total number of allocated memory blocks. - size_t num_all_blocks; - - /// Number of free memory blocks. - size_t num_free_blocks; -} coarse_memory_provider_stats_t; - -umf_memory_provider_ops_t *umfCoarseMemoryProviderOps(void); - -// TODO use CTL -coarse_memory_provider_stats_t -umfCoarseMemoryProviderGetStats(umf_memory_provider_handle_t provider); - -/// @brief Create default params for the coarse memory provider -static inline coarse_memory_provider_params_t -umfCoarseMemoryProviderParamsDefault(void) { - coarse_memory_provider_params_t coarse_memory_provider_params; - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - return coarse_memory_provider_params; -} - -#ifdef __cplusplus -} -#endif - -#endif // UMF_COARSE_PROVIDER_H diff --git a/include/umf/providers/provider_fixed_memory.h b/include/umf/providers/provider_fixed_memory.h new file mode 100644 index 0000000000..2351faf312 --- /dev/null +++ b/include/umf/providers/provider_fixed_memory.h @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#ifndef UMF_FIXED_MEMORY_PROVIDER_H +#define UMF_FIXED_MEMORY_PROVIDER_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/// @cond +#define UMF_FIXED_RESULTS_START_FROM 4000 +/// @endcond + +struct umf_fixed_memory_provider_params_t; + +typedef struct umf_fixed_memory_provider_params_t + *umf_fixed_memory_provider_params_handle_t; + +/// @brief Create a struct to store parameters of the Fixed Memory Provider. +/// @param hParams [out] handle to the newly created parameters struct. +/// @param ptr [in] pointer to the memory region. +/// @param size [in] size of the memory region in bytes. +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfFixedMemoryProviderParamsCreate( + umf_fixed_memory_provider_params_handle_t *hParams, void *ptr, size_t size); + +/// @brief Set the memory region in params struct. Overwrites the previous value. +/// It provides an ability to use the same instance of params to create multiple +/// instances of the provider for different memory regions. +/// @param hParams [in] handle to the parameters of the Fixed Memory Provider. +/// @param ptr [in] pointer to the memory region. +/// @param size [in] size of the memory region in bytes. +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfFixedMemoryProviderParamsSetMemory( + umf_fixed_memory_provider_params_handle_t hParams, void *ptr, size_t size); + +/// @brief Destroy parameters struct. +/// @param hParams [in] handle to the parameters of the Fixed Memory Provider. +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfFixedMemoryProviderParamsDestroy( + umf_fixed_memory_provider_params_handle_t hParams); + +/// @brief Retrieve the operations structure for the Fixed Memory Provider. +/// @return Pointer to the umf_memory_provider_ops_t structure. +umf_memory_provider_ops_t *umfFixedMemoryProviderOps(void); + +/// @brief Fixed Memory Provider operation results +typedef enum umf_fixed_memory_provider_native_error { + UMF_FIXED_RESULT_SUCCESS = UMF_FIXED_RESULTS_START_FROM, ///< Success + UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED, ///< Force purging failed +} umf_fixed_memory_provider_native_error_t; + +#ifdef __cplusplus +} +#endif + +#endif /* UMF_FIXED_MEMORY_PROVIDER_H */ diff --git a/include/umf/providers/provider_level_zero.h b/include/umf/providers/provider_level_zero.h index f760c57244..df6dd7364c 100644 --- a/include/umf/providers/provider_level_zero.h +++ b/include/umf/providers/provider_level_zero.h @@ -68,6 +68,21 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( umf_level_zero_memory_provider_params_handle_t hParams, ze_device_handle_t *hDevices, uint32_t deviceCount); +typedef enum umf_level_zero_memory_provider_free_policy_t { + UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_DEFAULT = + 0, ///< Free memory immediately. Default. + UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_BLOCKING_FREE, ///< Blocks until all commands using the memory are complete before freeing. + UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_DEFER_FREE, ///< Schedules the memory to be freed but does not free immediately. +} umf_level_zero_memory_provider_free_policy_t; + +/// @brief Set the memory free policy. +/// @param hParams handle to the parameters of the Level Zero Memory Provider. +/// @param policy memory free policy. +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfLevelZeroMemoryProviderParamsSetFreePolicy( + umf_level_zero_memory_provider_params_handle_t hParams, + umf_level_zero_memory_provider_free_policy_t policy); + umf_memory_provider_ops_t *umfLevelZeroMemoryProviderOps(void); #ifdef __cplusplus diff --git a/include/umf/providers/provider_os_memory.h b/include/umf/providers/provider_os_memory.h index a6bf43a7d9..90455cad19 100644 --- a/include/umf/providers/provider_os_memory.h +++ b/include/umf/providers/provider_os_memory.h @@ -1,9 +1,11 @@ /* - * Copyright (C) 2022-2024 Intel Corporation + * + * Copyright (C) 2022-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -*/ + * + */ #ifndef UMF_OS_MEMORY_PROVIDER_H #define UMF_OS_MEMORY_PROVIDER_H @@ -23,7 +25,7 @@ extern "C" { /// Not every mode is supported on every system. typedef enum umf_numa_mode_t { /// Default binding mode. Actual binding policy is system-specific. On - /// linux this corresponds to MPOL_DEFAULT. If this mode is specified, + /// Linux this corresponds to MPOL_DEFAULT. If this mode is specified, /// nodemask must be NULL and maxnode must be 0. UMF_NUMA_MODE_DEFAULT, diff --git a/scripts/README.md b/scripts/README.md deleted file mode 100644 index e3a9ed533f..0000000000 --- a/scripts/README.md +++ /dev/null @@ -1,5 +0,0 @@ -The documentation HTML files are generated using the following dependencies: - * [Python](https://www.python.org/downloads/) at least v3.8 - * [Doxygen](http://www.doxygen.nl/) at least v1.9.1 - - To generate files run the `generate_docs.py` script from the `scripts` directory. Files will be generated to the `docs/html` directory relative to the main directory of this repository. diff --git a/scripts/check_license/check_headers.sh b/scripts/check_license/check_headers.sh new file mode 100755 index 0000000000..aeb90e7a28 --- /dev/null +++ b/scripts/check_license/check_headers.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# Copyright (C) 2016-2025 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# check-headers.sh - check copyright and license in source files + +SELF=$0 + +function usage() { + echo "Usage: $SELF [-h|-v|-a|-d]" + echo " -h, --help this help message" + echo " -v, --verbose verbose mode" + echo " -a, --all check all files (only modified files are checked by default)" + echo " -d, --update_dates change Copyright dates in all analyzed files (rather not use with -a)" +} + +if [ "$#" -lt 2 ]; then + usage >&2 + exit 2 +fi + +SOURCE_ROOT=$1 +shift +LICENSE=$1 +shift + +PATTERN=`mktemp` +TMP=`mktemp` +TMP2=`mktemp` +TEMPFILE=`mktemp` +rm -f $PATTERN $TMP $TMP2 + +if [ "$1" == "-h" -o "$1" == "--help" ]; then + usage + exit 0 +fi + +export GIT="git -C ${SOURCE_ROOT}" +$GIT rev-parse || exit 1 + +if [ -f $SOURCE_ROOT/.git/shallow ]; then + SHALLOW_CLONE=1 + echo + echo "Warning: This is a shallow clone. Checking dates in copyright headers" + echo " will be skipped in case of files that have no history." + echo +else + SHALLOW_CLONE=0 +fi + +VERBOSE=0 +CHECK_ALL=0 +UPDATE_DATES=0 +while [ "$1" != "" ]; do + case $1 in + -v|--verbose) + VERBOSE=1 + ;; + -a|--all) + CHECK_ALL=1 + ;; + -d|--update_dates) + UPDATE_DATES=1 + ;; + esac + shift +done + +if [ $CHECK_ALL -eq 0 ]; then + CURRENT_COMMIT=$($GIT --no-pager log --pretty=%H -1) + MERGE_BASE=$($GIT merge-base HEAD origin/main 2>/dev/null) + [ -z $MERGE_BASE ] && \ + MERGE_BASE=$($GIT --no-pager log --pretty="%cN:%H" | grep GitHub 2>/dev/null | head -n1 | cut -d: -f2) + [ -z $MERGE_BASE -o "$CURRENT_COMMIT" = "$MERGE_BASE" ] && \ + CHECK_ALL=1 +fi + +if [ $CHECK_ALL -eq 1 ]; then + echo "INFO: Checking copyright headers of all files..." + GIT_COMMAND="ls-tree -r --name-only HEAD" +else + echo "INFO: Checking copyright headers of modified files only..." + GIT_COMMAND="diff --name-only $MERGE_BASE $CURRENT_COMMIT" +fi + +FILES=$($GIT $GIT_COMMAND | ${SOURCE_ROOT}/scripts/check_license/file-exceptions.sh) + +RV=0 +for file in $FILES ; do + if [ $VERBOSE -eq 1 ]; then + echo "Checking file: $file" + fi + # The src_path is a path which should be used in every command except git. + # git is called with -C flag so filepaths should be relative to SOURCE_ROOT + src_path="${SOURCE_ROOT}/$file" + [ ! -f $src_path ] && continue + # ensure that file is UTF-8 encoded + ENCODING=`file -b --mime-encoding $src_path` + iconv -f $ENCODING -t "UTF-8" $src_path > $TEMPFILE + + if ! grep -q "SPDX-License-Identifier: $LICENSE" $src_path; then + echo >&2 "error: no $LICENSE SPDX tag in file: $src_path" + RV=1 + fi + + if [ $SHALLOW_CLONE -eq 0 ]; then + $GIT log --no-merges --format="%ai %aE" -- $file | sort > $TMP + else + # mark the grafted commits (commits with no parents) + $GIT log --no-merges --format="%ai %aE grafted-%p-commit" -- $file | sort > $TMP + fi + + # skip checking dates for non-Intel commits + [[ ! $(tail -n1 $TMP) =~ "@intel.com" ]] && continue + + # skip checking dates for new files + [ $(cat $TMP | wc -l) -le 1 ] && continue + + # grep out the grafted commits (commits with no parents) + # and skip checking dates for non-Intel commits + grep -v -e "grafted--commit" $TMP | grep -e "@intel.com" > $TMP2 + + [ $(cat $TMP2 | wc -l) -eq 0 ] && continue + + FIRST=`head -n1 $TMP2` + LAST=` tail -n1 $TMP2` + + YEARS=$(sed ' +/.*Copyright (C) [0-9-]\+ Intel Corporation/!d +s/.*Copyright (C) \([0-9]\+\)-\([0-9]\+\).*/\1-\2/ +s/.*Copyright (C) \([0-9]\+\).*/\1/' "$src_path") + if [ -z "$YEARS" ]; then + echo >&2 "No copyright years in $src_path" + RV=1 + continue + fi + + HEADER_FIRST=`echo $YEARS | cut -d"-" -f1` + HEADER_LAST=` echo $YEARS | cut -d"-" -f2` + + COMMIT_FIRST=`echo $FIRST | cut -d"-" -f1` + COMMIT_LAST=` echo $LAST | cut -d"-" -f1` + + if [ "$COMMIT_FIRST" != "" -a "$COMMIT_LAST" != "" ]; then + if [ "$COMMIT_FIRST" -lt "$HEADER_FIRST" ]; then + RV=1 + fi + + if [[ -n "$COMMIT_FIRST" && -n "$COMMIT_LAST" ]]; then + if [[ $HEADER_FIRST -le $COMMIT_FIRST ]]; then + if [[ $HEADER_LAST -eq $COMMIT_LAST ]]; then + continue + else + NEW="$HEADER_FIRST-$COMMIT_LAST" + if [[ ${UPDATE_DATES} -eq 1 ]]; then + echo "Updating copyright date in $src_path: $YEARS -> $NEW" + sed -i "s/Copyright (C) ${YEARS}/Copyright (C) ${NEW}/g" "${src_path}" + else + echo "$file:1: error: wrong copyright date: (is: $YEARS, should be: $NEW)" >&2 + RV=1 + fi + fi + else + if [[ $COMMIT_FIRST -eq $COMMIT_LAST ]]; then + NEW=$COMMIT_LAST + else + NEW=$COMMIT_FIRST-$COMMIT_LAST + fi + + if [[ "$YEARS" == "$NEW" ]]; then + continue + else + if [[ ${UPDATE_DATES} -eq 1 ]]; then + echo "Updating copyright date in $src_path: $YEARS -> $NEW" + sed -i "s/Copyright (C) ${YEARS}/Copyright (C) ${NEW}/g" "${src_path}" + else + echo "$file:1: error: wrong copyright date: (is: $YEARS, should be: $NEW)" >&2 + RV=1 + fi + fi + fi + fi + else + echo "error: unknown commit dates in file: $file" >&2 + RV=1 + fi +done +rm -f $TMP $TMP2 $TEMPFILE + +# check if error found +if [ $RV -eq 0 ]; then + echo "Copyright headers are OK." +else + echo "Error(s) in copyright headers found!" >&2 +fi +exit $RV diff --git a/scripts/check_license/file-exceptions.sh b/scripts/check_license/file-exceptions.sh new file mode 100755 index 0000000000..10c5560614 --- /dev/null +++ b/scripts/check_license/file-exceptions.sh @@ -0,0 +1,38 @@ +#!/bin/sh -e +# Copyright (C) 2025 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# You can add an exception file +# list for license and copyright check +grep -v -E -e 'benchmark/ubench.h' \ + -e 'ChangeLog' \ + -e 'CODEOWNERS$' \ + -e 'docs/assets/.*' \ + -e 'docs/config/spelling_exceptions.txt' \ + -e 'docs/config/conf.py' \ + -e 'docs/config/Doxyfile' \ + -e 'include/umf/proxy_lib_new_delete.h' \ + -e 'LICENSE.TXT' \ + -e 'licensing/third-party-programs.txt' \ + -e 'scripts/assets/images/.*' \ + -e 'scripts/qemu/requirements.txt' \ + -e 'src/uthash/.*' \ + -e 'src/uthash/utlist.h' \ + -e 'src/uthash/uthash.h' \ + -e 'test/ctl/config.txt' \ + -e 'test/supp/.*' \ + -e 'third_party/requirements.txt' \ + -e '.clang-format$' \ + -e '.cmake-format$' \ + -e '.cmake.in$' \ + -e '.gitignore' \ + -e '.json$' \ + -e '.mailmap' \ + -e '.md$' \ + -e '.patch$' \ + -e '.rst$' \ + -e '.spellcheck-conf.toml' \ + -e '.trivyignore' \ + -e '.xml$' \ + -e '.yml$' diff --git a/scripts/qemu/run-build.sh b/scripts/qemu/run-build.sh index 06d6043f6b..c6314153c6 100755 --- a/scripts/qemu/run-build.sh +++ b/scripts/qemu/run-build.sh @@ -14,13 +14,14 @@ pwd echo password | sudo -Sk apt-get update echo password | sudo -Sk apt-get install -y git cmake gcc g++ pkg-config \ - numactl libnuma-dev hwloc libhwloc-dev libjemalloc-dev libtbb-dev valgrind lcov + numactl libnuma-dev hwloc libhwloc-dev libtbb-dev valgrind lcov mkdir build cd build cmake .. \ -DCMAKE_BUILD_TYPE=Debug \ + -DUMF_QEMU_BUILD=1 \ -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON \ -DUMF_BUILD_CUDA_PROVIDER=ON \ -DUMF_FORMAT_CODE_STYLE=OFF \ diff --git a/scripts/qemu/run-tests.sh b/scripts/qemu/run-tests.sh index 9d855590ba..341e2f9ab8 100755 --- a/scripts/qemu/run-tests.sh +++ b/scripts/qemu/run-tests.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2024-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -23,8 +23,6 @@ UMF_DIR=$(pwd) # Drop caches, restores free memory on NUMA nodes echo password | sudo sync; echo password | sudo sh -c "/usr/bin/echo 3 > /proc/sys/vm/drop_caches" -# Set ptrace value for IPC test -echo password | sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" numactl -H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 57050e8276..c0072be7e4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (C) 2023-2024 Intel Corporation +# Copyright (C) 2023-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -27,18 +27,21 @@ endforeach() # # TODO: Cleanup the compile definitions across all the CMake files set(UMF_COMMON_COMPILE_DEFINITIONS - UMF_VERSION=${UMF_VERSION} + ${UMF_COMMON_COMPILE_DEFINITIONS} UMF_VERSION=${UMF_VERSION} UMF_ALL_CMAKE_VARIABLES="${UMF_ALL_CMAKE_VARIABLES}") -add_subdirectory(utils) - -set(UMF_LIBS $) - set(BA_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/base_alloc/base_alloc.c ${CMAKE_CURRENT_SOURCE_DIR}/base_alloc/base_alloc_linear.c ${CMAKE_CURRENT_SOURCE_DIR}/base_alloc/base_alloc_global.c) +add_subdirectory(utils) +add_subdirectory(coarse) + +set(UMF_LIBS $ $) + +set(CTL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ctl/ctl.c) + if(LINUX) set(BA_SOURCES ${BA_SOURCES} ${CMAKE_CURRENT_SOURCE_DIR}/base_alloc/base_alloc_linux.c) @@ -58,6 +61,7 @@ set(HWLOC_DEPENDENT_SOURCES topology.c) set(UMF_SOURCES ${BA_SOURCES} + ${CTL_SOURCES} libumf.c ipc.c ipc_cache.c @@ -72,18 +76,29 @@ set(UMF_SOURCES memspaces/memspace_highest_bandwidth.c memspaces/memspace_lowest_latency.c memspaces/memspace_numa.c - provider/provider_coarse.c provider/provider_cuda.c provider/provider_devdax_memory.c provider/provider_file_memory.c + provider/provider_fixed_memory.c provider/provider_level_zero.c provider/provider_os_memory.c provider/provider_tracking.c critnib/critnib.c ravl/ravl.c pool/pool_proxy.c + pool/pool_jemalloc.c pool/pool_scalable.c) +if(UMF_POOL_JEMALLOC_ENABLED) + set(UMF_LIBS ${UMF_LIBS} ${JEMALLOC_LIBRARIES}) + set(UMF_PRIVATE_LIBRARY_DIRS ${UMF_PRIVATE_LIBRARY_DIRS} + ${JEMALLOC_LIBRARY_DIRS}) + set(UMF_PRIVATE_INCLUDE_DIRS ${UMF_PRIVATE_INCLUDE_DIRS} + ${JEMALLOC_INCLUDE_DIRS}) + set(UMF_COMMON_COMPILE_DEFINITIONS ${UMF_COMMON_COMPILE_DEFINITIONS} + "UMF_POOL_JEMALLOC_ENABLED=1") +endif() + if(NOT UMF_DISABLE_HWLOC) set(UMF_SOURCES ${UMF_SOURCES} ${HWLOC_DEPENDENT_SOURCES} memtargets/memtarget_numa.c) @@ -159,13 +174,31 @@ else() LIBS ${UMF_LIBS}) endif() +target_include_directories(umf PRIVATE ${UMF_PRIVATE_INCLUDE_DIRS}) +target_link_directories(umf PRIVATE ${UMF_PRIVATE_LIBRARY_DIRS}) +target_compile_definitions(umf PRIVATE ${UMF_COMMON_COMPILE_DEFINITIONS}) + +add_dependencies(umf coarse) + if(UMF_LINK_HWLOC_STATICALLY) add_dependencies(umf ${UMF_HWLOC_NAME}) + # On Darwin, link with the IOKit and Foundation frameworks, if they are + # available in the system. This is to comply with hwloc which links these, + # if available. There is no option to disable these frameworks on Darwin + # hwloc builds. + if(MACOSX) + find_library(IOKIT_LIBRARY IOKit) + find_library(FOUNDATION_LIBRARY Foundation) + if(IOKIT_LIBRARY OR FOUNDATION_LIBRARY) + target_link_libraries(umf PRIVATE ${IOKIT_LIBRARY} + ${FOUNDATION_LIBRARY}) + endif() + endif() endif() -target_link_directories(umf PRIVATE ${UMF_PRIVATE_LIBRARY_DIRS}) - -target_compile_definitions(umf PRIVATE ${UMF_COMMON_COMPILE_DEFINITIONS}) +if(NOT WINDOWS AND UMF_POOL_JEMALLOC_ENABLED) + add_dependencies(umf jemalloc) +endif() if(UMF_BUILD_LEVEL_ZERO_PROVIDER) if(LINUX) diff --git a/src/base_alloc/base_alloc.c b/src/base_alloc/base_alloc.c index 209ace7fe3..6f975307dc 100644 --- a/src/base_alloc/base_alloc.c +++ b/src/base_alloc/base_alloc.c @@ -303,7 +303,13 @@ void umf_ba_destroy(umf_ba_pool_t *pool) { #ifndef NDEBUG ba_debug_checks(pool); if (pool->metadata.n_allocs) { - LOG_ERR("pool->metadata.n_allocs = %zu", pool->metadata.n_allocs); + LOG_ERR("number of base allocator memory leaks: %zu", + pool->metadata.n_allocs); + +#ifdef UMF_DEVELOPER_MODE + assert(pool->metadata.n_allocs == 0 && + "memory leaks in base allocator occurred"); +#endif } #endif /* NDEBUG */ diff --git a/src/coarse/CMakeLists.txt b/src/coarse/CMakeLists.txt new file mode 100644 index 0000000000..8806b6b557 --- /dev/null +++ b/src/coarse/CMakeLists.txt @@ -0,0 +1,26 @@ +# Copyright (C) 2024 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +include(${UMF_CMAKE_SOURCE_DIR}/cmake/helpers.cmake) + +set(COARSE_SOURCES coarse.c ../ravl/ravl.c) + +if(UMF_BUILD_SHARED_LIBRARY) + set(COARSE_EXTRA_SRCS ${BA_SOURCES}) + set(COARSE_EXTRA_LIBS $) +endif() + +add_umf_library( + NAME coarse + TYPE STATIC + SRCS ${COARSE_SOURCES} ${COARSE_EXTRA_SRCS} + LIBS ${COARSE_EXTRA_LIBS}) + +target_include_directories( + coarse + PRIVATE $ + $ + $) + +add_library(${PROJECT_NAME}::coarse ALIAS coarse) diff --git a/src/coarse/coarse.c b/src/coarse/coarse.c new file mode 100644 index 0000000000..0ce4ded3d3 --- /dev/null +++ b/src/coarse/coarse.c @@ -0,0 +1,1367 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include +#include +#include +#include +#include +#include + +#include + +#include "base_alloc_global.h" +#include "coarse.h" +#include "libumf.h" +#include "ravl.h" +#include "utils_common.h" +#include "utils_concurrency.h" +#include "utils_log.h" + +#ifdef _WIN32 +UTIL_ONCE_FLAG Log_initialized = UTIL_ONCE_FLAG_INIT; +#else +void __attribute__((constructor)) coarse_init(void) { utils_log_init(); } +void __attribute__((destructor)) coarse_destroy(void) {} +#endif /* _WIN32 */ + +typedef struct coarse_t { + // handle of the memory provider + void *provider; + + // coarse callbacks + coarse_callbacks_t cb; + + // memory allocation strategy + coarse_strategy_t allocation_strategy; + + // page size of the memory provider + size_t page_size; + + // all_blocks - tree of all blocks - sorted by an address of data + struct ravl *all_blocks; + + // free_blocks - tree of free blocks - sorted by a size of data, + // each node contains a pointer (ravl_free_blocks_head_t) + // to the head of the list of free blocks of the same size + struct ravl *free_blocks; + + struct utils_mutex_t lock; + + // statistics + size_t used_size; + size_t alloc_size; +} coarse_t; + +typedef struct ravl_node ravl_node_t; + +typedef enum check_free_blocks_t { + CHECK_ONLY_THE_FIRST_BLOCK = 0, + CHECK_ALL_BLOCKS_OF_SIZE, +} check_free_blocks_t; + +typedef struct block_t { + size_t size; + unsigned char *data; + bool used; + + // Node in the list of free blocks of the same size pointing to this block. + // The list is located in the (coarse->free_blocks) RAVL tree. + struct ravl_free_blocks_elem_t *free_list_ptr; +} block_t; + +// A general node in a RAVL tree. +// 1) coarse->all_blocks RAVL tree (tree of all blocks - sorted by an address of data): +// key - pointer (block_t->data) to the beginning of the block data +// value - pointer (block_t) to the block of the allocation +// 2) coarse->free_blocks RAVL tree (tree of free blocks - sorted by a size of data): +// key - size of the allocation (block_t->size) +// value - pointer (ravl_free_blocks_head_t) to the head of the list of free blocks of the same size +typedef struct ravl_data_t { + uintptr_t key; + void *value; +} ravl_data_t; + +// The head of the list of free blocks of the same size. +typedef struct ravl_free_blocks_head_t { + struct ravl_free_blocks_elem_t *head; +} ravl_free_blocks_head_t; + +// The node of the list of free blocks of the same size +typedef struct ravl_free_blocks_elem_t { + struct block_t *block; + struct ravl_free_blocks_elem_t *next; + struct ravl_free_blocks_elem_t *prev; +} ravl_free_blocks_elem_t; + +// The compare function of a RAVL tree +static int coarse_ravl_comp(const void *lhs, const void *rhs) { + const ravl_data_t *lhs_ravl = (const ravl_data_t *)lhs; + const ravl_data_t *rhs_ravl = (const ravl_data_t *)rhs; + + if (lhs_ravl->key < rhs_ravl->key) { + return -1; + } + + if (lhs_ravl->key > rhs_ravl->key) { + return 1; + } + + // lhs_ravl->key == rhs_ravl->key + return 0; +} + +static inline block_t *get_node_block(ravl_node_t *node) { + ravl_data_t *node_data = ravl_data(node); + assert(node_data); + assert(node_data->value); + return node_data->value; +} + +static inline ravl_node_t *get_node_prev(ravl_node_t *node) { + return ravl_node_predecessor(node); +} + +static inline ravl_node_t *get_node_next(ravl_node_t *node) { + return ravl_node_successor(node); +} + +#ifndef NDEBUG +static block_t *get_block_prev(ravl_node_t *node) { + ravl_node_t *ravl_prev = ravl_node_predecessor(node); + if (!ravl_prev) { + return NULL; + } + + return get_node_block(ravl_prev); +} + +static block_t *get_block_next(ravl_node_t *node) { + ravl_node_t *ravl_next = ravl_node_successor(node); + if (!ravl_next) { + return NULL; + } + + return get_node_block(ravl_next); +} +#endif /* NDEBUG */ + +// The functions "coarse_ravl_*" handles the coarse->all_blocks list of blocks +// sorted by a pointer (block_t->data) to the beginning of the block data. +// +// coarse_ravl_add_new - allocate and add a new block to the tree +// and link this block to the next and the previous one. +static block_t *coarse_ravl_add_new(struct ravl *rtree, unsigned char *data, + size_t size, ravl_node_t **node) { + assert(rtree); + assert(data); + assert(size); + + // TODO add valgrind annotations + block_t *block = umf_ba_global_alloc(sizeof(*block)); + if (block == NULL) { + return NULL; + } + + block->data = data; + block->size = size; + block->free_list_ptr = NULL; + + ravl_data_t rdata = {(uintptr_t)block->data, block}; + assert(NULL == ravl_find(rtree, &data, RAVL_PREDICATE_EQUAL)); + int ret = ravl_emplace_copy(rtree, &rdata); + if (ret) { + umf_ba_global_free(block); + return NULL; + } + + ravl_node_t *new_node = ravl_find(rtree, &rdata, RAVL_PREDICATE_EQUAL); + assert(NULL != new_node); + + if (node) { + *node = new_node; + } + + return block; +} + +// coarse_ravl_find_node - find the node in the tree +static ravl_node_t *coarse_ravl_find_node(struct ravl *rtree, void *ptr) { + ravl_data_t data = {(uintptr_t)ptr, NULL}; + return ravl_find(rtree, &data, RAVL_PREDICATE_EQUAL); +} + +// coarse_ravl_rm - remove the block from the tree +static block_t *coarse_ravl_rm(struct ravl *rtree, void *ptr) { + ravl_data_t data = {(uintptr_t)ptr, NULL}; + ravl_node_t *node; + node = ravl_find(rtree, &data, RAVL_PREDICATE_EQUAL); + if (node) { + ravl_data_t *node_data = ravl_data(node); + assert(node_data); + block_t *block = node_data->value; + assert(block); + ravl_remove(rtree, node); + assert(NULL == ravl_find(rtree, &data, RAVL_PREDICATE_EQUAL)); + return block; + } + return NULL; +} + +// The functions "node_list_*" handle lists of free blocks of the same size. +// The heads (ravl_free_blocks_head_t) of those lists are stored in nodes of +// the coarse->free_blocks RAVL tree. +// +// node_list_add - add a free block to the list of free blocks of the same size +static ravl_free_blocks_elem_t * +node_list_add(ravl_free_blocks_head_t *head_node, struct block_t *block) { + assert(head_node); + assert(block); + + ravl_free_blocks_elem_t *node = umf_ba_global_alloc(sizeof(*node)); + if (node == NULL) { + return NULL; + } + + if (head_node->head) { + head_node->head->prev = node; + } + + node->block = block; + node->next = head_node->head; + node->prev = NULL; + head_node->head = node; + + return node; +} + +// node_list_rm - remove the given free block from the list of free blocks of the same size +static block_t *node_list_rm(ravl_free_blocks_head_t *head_node, + ravl_free_blocks_elem_t *node) { + assert(head_node); + assert(node); + assert(head_node->head); + + if (node == head_node->head) { + assert(node->prev == NULL); + head_node->head = node->next; + } + + ravl_free_blocks_elem_t *node_next = node->next; + ravl_free_blocks_elem_t *node_prev = node->prev; + if (node_next) { + node_next->prev = node_prev; + } + + if (node_prev) { + node_prev->next = node_next; + } + + struct block_t *block = node->block; + block->free_list_ptr = NULL; + umf_ba_global_free(node); + + return block; +} + +// node_list_rm_first - remove the first free block from the list of free blocks of the same size only if it can be properly aligned +static block_t *node_list_rm_first(ravl_free_blocks_head_t *head_node, + size_t alignment) { + assert(head_node); + assert(head_node->head); + + ravl_free_blocks_elem_t *node = head_node->head; + assert(node->prev == NULL); + struct block_t *block = node->block; + + if (IS_NOT_ALIGNED(block->size, alignment)) { + return NULL; + } + + if (node->next) { + node->next->prev = NULL; + } + + head_node->head = node->next; + block->free_list_ptr = NULL; + umf_ba_global_free(node); + + return block; +} + +// node_list_rm_with_alignment - remove the first free block with the correct alignment from the list of free blocks of the same size +static block_t *node_list_rm_with_alignment(ravl_free_blocks_head_t *head_node, + size_t alignment) { + assert(head_node); + assert(head_node->head); + + assert(((ravl_free_blocks_elem_t *)head_node->head)->prev == NULL); + + ravl_free_blocks_elem_t *node; + for (node = head_node->head; node != NULL; node = node->next) { + if (IS_ALIGNED(node->block->size, alignment)) { + return node_list_rm(head_node, node); + } + } + + return NULL; +} + +// The functions "free_blocks_*" handle the coarse->free_blocks RAVL tree +// sorted by a size of the allocation (block_t->size). +// This is a tree of heads (ravl_free_blocks_head_t) of lists of free blocks of the same size. +// +// free_blocks_add - add a free block to the list of free blocks of the same size +static int free_blocks_add(struct ravl *free_blocks, block_t *block) { + ravl_free_blocks_head_t *head_node = NULL; + int rv; + + ravl_data_t head_node_data = {(uintptr_t)block->size, NULL}; + ravl_node_t *node; + node = ravl_find(free_blocks, &head_node_data, RAVL_PREDICATE_EQUAL); + if (node) { + ravl_data_t *node_data = ravl_data(node); + assert(node_data); + head_node = node_data->value; + assert(head_node); + } else { // no head_node + head_node = umf_ba_global_alloc(sizeof(*head_node)); + if (!head_node) { + return -1; + } + + head_node->head = NULL; + + ravl_data_t data = {(uintptr_t)block->size, head_node}; + rv = ravl_emplace_copy(free_blocks, &data); + if (rv) { + umf_ba_global_free(head_node); + return -1; + } + } + + block->free_list_ptr = node_list_add(head_node, block); + if (!block->free_list_ptr) { + return -1; // out of memory + } + + assert(block->free_list_ptr->block->size == block->size); + + return 0; +} + +// free_blocks_rm_ge - remove the first free block of a size greater or equal to the given size only if it can be properly aligned +// If it was the last block, the head node is freed and removed from the tree. +// It is used during memory allocation (looking for a free block). +static block_t *free_blocks_rm_ge(struct ravl *free_blocks, size_t size, + size_t alignment, + check_free_blocks_t check_blocks) { + ravl_data_t data = {(uintptr_t)size, NULL}; + ravl_node_t *node; + node = ravl_find(free_blocks, &data, RAVL_PREDICATE_GREATER_EQUAL); + if (!node) { + return NULL; + } + + ravl_data_t *node_data = ravl_data(node); + assert(node_data); + assert(node_data->key >= size); + + ravl_free_blocks_head_t *head_node = node_data->value; + assert(head_node); + + block_t *block = NULL; + switch (check_blocks) { + case CHECK_ONLY_THE_FIRST_BLOCK: + block = node_list_rm_first(head_node, alignment); + break; + case CHECK_ALL_BLOCKS_OF_SIZE: + block = node_list_rm_with_alignment(head_node, alignment); + break; + } + + if (head_node->head == NULL) { + umf_ba_global_free(head_node); + ravl_remove(free_blocks, node); + } + + return block; +} + +// free_blocks_rm_node - remove the free block pointed by the given node. +// If it was the last block, the head node is freed and removed from the tree. +// It is used during merging free blocks and destroying the coarse->free_blocks tree. +static block_t *free_blocks_rm_node(struct ravl *free_blocks, + ravl_free_blocks_elem_t *node) { + assert(free_blocks); + assert(node); + size_t size = node->block->size; + ravl_data_t data = {(uintptr_t)size, NULL}; + ravl_node_t *ravl_node; + ravl_node = ravl_find(free_blocks, &data, RAVL_PREDICATE_EQUAL); + assert(ravl_node); + + ravl_data_t *node_data = ravl_data(ravl_node); + assert(node_data); + assert(node_data->key == size); + + ravl_free_blocks_head_t *head_node = node_data->value; + assert(head_node); + + block_t *block = node_list_rm(head_node, node); + + if (head_node->head == NULL) { + umf_ba_global_free(head_node); + ravl_remove(free_blocks, ravl_node); + } + + return block; +} + +// user_block_merge - merge two blocks from one of two lists of user blocks: all_blocks or free_blocks +static umf_result_t user_block_merge(coarse_t *coarse, ravl_node_t *node1, + ravl_node_t *node2, bool used, + ravl_node_t **merged_node) { + assert(node1); + assert(node2); + assert(node1 == get_node_prev(node2)); + assert(node2 == get_node_next(node1)); + assert(merged_node); + + *merged_node = NULL; + + struct ravl *all_blocks = coarse->all_blocks; + struct ravl *free_blocks = coarse->free_blocks; + + block_t *block1 = get_node_block(node1); + block_t *block2 = get_node_block(node2); + assert(block1->data < block2->data); + + bool same_used = ((block1->used == used) && (block2->used == used)); + bool contignous_data = (block1->data + block1->size == block2->data); + + // check if blocks can be merged + if (!same_used || !contignous_data) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + // check if blocks can be merged + umf_result_t umf_result = + coarse->cb.merge(coarse->provider, block1->data, block2->data, + block1->size + block2->size); + if (umf_result != UMF_RESULT_SUCCESS) { + LOG_ERR("coarse_merge_cb(lowPtr=%p, highPtr=%p, totalSize=%zu) failed", + (void *)block1->data, (void *)block2->data, + block1->size + block2->size); + return umf_result; + } + + if (block1->free_list_ptr) { + free_blocks_rm_node(free_blocks, block1->free_list_ptr); + block1->free_list_ptr = NULL; + } + + if (block2->free_list_ptr) { + free_blocks_rm_node(free_blocks, block2->free_list_ptr); + block2->free_list_ptr = NULL; + } + + // update the size + block1->size += block2->size; + + block_t *block_rm = coarse_ravl_rm(all_blocks, block2->data); + assert(block_rm == block2); + (void)block_rm; // WA for unused variable error + umf_ba_global_free(block2); + + *merged_node = node1; + + return UMF_RESULT_SUCCESS; +} + +// free_block_merge_with_prev - merge the given free block +// with the previous one if both are unused and have continuous data. +// Remove the merged block from the tree of free blocks. +static ravl_node_t *free_block_merge_with_prev(coarse_t *coarse, + ravl_node_t *node) { + ravl_node_t *node_prev = get_node_prev(node); + if (!node_prev) { + return node; + } + + ravl_node_t *merged_node = NULL; + umf_result_t umf_result = + user_block_merge(coarse, node_prev, node, false, &merged_node); + if (umf_result != UMF_RESULT_SUCCESS) { + return node; + } + + assert(merged_node != NULL); + + return merged_node; +} + +// free_block_merge_with_next - merge the given free block +// with the next one if both are unused and have continuous data. +// Remove the merged block from the tree of free blocks. +static ravl_node_t *free_block_merge_with_next(coarse_t *coarse, + ravl_node_t *node) { + ravl_node_t *node_next = get_node_next(node); + if (!node_next) { + return node; + } + + ravl_node_t *merged_node = NULL; + umf_result_t umf_result = + user_block_merge(coarse, node, node_next, false, &merged_node); + if (umf_result != UMF_RESULT_SUCCESS) { + return node; + } + + assert(merged_node != NULL); + + return merged_node; +} + +#ifndef NDEBUG // begin of DEBUG code + +typedef struct debug_cb_args_t { + coarse_t *provider; + size_t sum_used; + size_t sum_blocks_size; + size_t num_all_blocks; + size_t num_free_blocks; +} debug_cb_args_t; + +static void debug_verify_all_blocks_cb(void *data, void *arg) { + assert(data); + assert(arg); + + ravl_data_t *node_data = data; + block_t *block = node_data->value; + assert(block); + + debug_cb_args_t *cb_args = (debug_cb_args_t *)arg; + coarse_t *provider = cb_args->provider; + + ravl_node_t *node = + ravl_find(provider->all_blocks, data, RAVL_PREDICATE_EQUAL); + assert(node); + + block_t *block_next = get_block_next(node); + block_t *block_prev = get_block_prev(node); + + cb_args->num_all_blocks++; + if (!block->used) { + cb_args->num_free_blocks++; + } + + assert(block->data); + assert(block->size > 0); + + // data addresses in the list are in ascending order + if (block_prev) { + assert(block_prev->data < block->data); + } + + if (block_next) { + assert(block->data < block_next->data); + } + + // two block's data should not overlap + if (block_next) { + assert((block->data + block->size) <= block_next->data); + } + + cb_args->sum_blocks_size += block->size; + if (block->used) { + cb_args->sum_used += block->size; + } +} + +static umf_result_t coarse_get_stats_no_lock(coarse_t *coarse, + coarse_stats_t *stats); + +static bool debug_check(coarse_t *provider) { + assert(provider); + + coarse_stats_t stats = {0}; + coarse_get_stats_no_lock(provider, &stats); + + debug_cb_args_t cb_args = {0}; + cb_args.provider = provider; + + // verify the all_blocks list + ravl_foreach(provider->all_blocks, debug_verify_all_blocks_cb, &cb_args); + + assert(cb_args.num_all_blocks == stats.num_all_blocks); + assert(cb_args.num_free_blocks == stats.num_free_blocks); + assert(cb_args.sum_used == provider->used_size); + assert(cb_args.sum_blocks_size == provider->alloc_size); + assert(provider->alloc_size >= provider->used_size); + + return true; +} +#endif /* NDEBUG */ // end of DEBUG code + +static umf_result_t coarse_add_used_block(coarse_t *coarse, void *addr, + size_t size) { + block_t *new_block = + coarse_ravl_add_new(coarse->all_blocks, addr, size, NULL); + if (new_block == NULL) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + new_block->used = true; + coarse->alloc_size += size; + coarse->used_size += size; + + return UMF_RESULT_SUCCESS; +} + +static void coarse_ravl_cb_rm_all_blocks_node(void *data, void *arg) { + assert(data); + assert(arg); + + coarse_t *coarse = (struct coarse_t *)arg; + ravl_data_t *node_data = data; + block_t *block = node_data->value; + assert(block); + + if (block->used) { +#ifndef NDEBUG + LOG_WARN("not freed block (addr: %p, size: %zu)", (void *)block->data, + block->size); +#endif + assert(coarse->used_size >= block->size); + coarse->used_size -= block->size; + } + + if (block->free_list_ptr) { + free_blocks_rm_node(coarse->free_blocks, block->free_list_ptr); + } + + if (coarse->cb.free) { + coarse->cb.free(coarse->provider, block->data, block->size); + } + + assert(coarse->alloc_size >= block->size); + coarse->alloc_size -= block->size; + + umf_ba_global_free(block); +} + +static umf_result_t can_provider_split(coarse_t *coarse, void *ptr, + size_t totalSize, size_t firstSize) { + // check if the block can be split + umf_result_t umf_result = + coarse->cb.split(coarse->provider, ptr, totalSize, firstSize); + if (umf_result != UMF_RESULT_SUCCESS) { + LOG_ERR( + "coarse_split_cb->(ptr=%p, totalSize = %zu = (%zu + %zu)) failed", + ptr, totalSize, firstSize, totalSize - firstSize); + } + + return umf_result; +} + +static umf_result_t create_aligned_block(coarse_t *coarse, size_t orig_size, + size_t alignment, block_t **current) { + (void)orig_size; // unused in the Release version + int rv; + + block_t *curr = *current; + + // In case of non-zero alignment create an aligned block what would be further used. + uintptr_t orig_data = (uintptr_t)curr->data; + uintptr_t aligned_data = ALIGN_UP(orig_data, alignment); + size_t padding = aligned_data - orig_data; + if (alignment > 0 && padding > 0) { + // check if block can be split by the upstream provider + umf_result_t umf_result = + can_provider_split(coarse, curr->data, curr->size, padding); + if (umf_result != UMF_RESULT_SUCCESS) { + return umf_result; + } + + block_t *aligned_block = + coarse_ravl_add_new(coarse->all_blocks, curr->data + padding, + curr->size - padding, NULL); + if (aligned_block == NULL) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + curr->used = false; + curr->size = padding; + + rv = free_blocks_add(coarse->free_blocks, curr); + if (rv) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + // use aligned block + *current = aligned_block; + assert((*current)->size >= orig_size); + } + + return UMF_RESULT_SUCCESS; +} + +// Split the current block and put the new block after the one that we use. +static umf_result_t split_current_block(coarse_t *coarse, block_t *curr, + size_t size) { + + // check if block can be split by the upstream provider + umf_result_t umf_result = + can_provider_split(coarse, curr->data, curr->size, size); + if (umf_result != UMF_RESULT_SUCCESS) { + return umf_result; + } + + ravl_node_t *new_node = NULL; + + block_t *new_block = coarse_ravl_add_new( + coarse->all_blocks, curr->data + size, curr->size - size, &new_node); + if (new_block == NULL) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + new_block->used = false; + + int rv = free_blocks_add(coarse->free_blocks, get_node_block(new_node)); + if (rv) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + return UMF_RESULT_SUCCESS; +} + +static block_t *find_free_block(struct ravl *free_blocks, size_t size, + size_t alignment, + coarse_strategy_t allocation_strategy) { + block_t *block; + size_t new_size = size + alignment; + + switch (allocation_strategy) { + case UMF_COARSE_MEMORY_STRATEGY_FASTEST: + // Always allocate a free block of the (size + alignment) size + // and later cut out the properly aligned part leaving two remaining parts. + if (new_size < size) { + LOG_ERR("arithmetic overflow (size + alignment)"); + return NULL; + } + + return free_blocks_rm_ge(free_blocks, new_size, 0, + CHECK_ONLY_THE_FIRST_BLOCK); + + case UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE: + // First check if the first free block of the 'size' size has the correct alignment. + block = free_blocks_rm_ge(free_blocks, size, alignment, + CHECK_ONLY_THE_FIRST_BLOCK); + if (block) { + return block; + } + + if (new_size < size) { + LOG_ERR("arithmetic overflow (size + alignment)"); + return NULL; + } + + // If not, use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. + return free_blocks_rm_ge(free_blocks, new_size, 0, + CHECK_ONLY_THE_FIRST_BLOCK); + + case UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE: + // First look through all free blocks of the 'size' size + // and choose the first one with the correct alignment. + block = free_blocks_rm_ge(free_blocks, size, alignment, + CHECK_ALL_BLOCKS_OF_SIZE); + if (block) { + return block; + } + + if (new_size < size) { + LOG_ERR("arithmetic overflow (size + alignment)"); + return NULL; + } + + // If none of them had the correct alignment, + // use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. + return free_blocks_rm_ge(free_blocks, new_size, 0, + CHECK_ONLY_THE_FIRST_BLOCK); + } + + return NULL; +} + +static int free_blocks_re_add(coarse_t *coarse, block_t *block) { + assert(coarse); + + ravl_node_t *node = coarse_ravl_find_node(coarse->all_blocks, block->data); + assert(node); + + // merge with prev and/or next block if they are unused and have continuous data + node = free_block_merge_with_prev(coarse, node); + node = free_block_merge_with_next(coarse, node); + + return free_blocks_add(coarse->free_blocks, get_node_block(node)); +} + +static void ravl_cb_count(void *data, void *arg) { + assert(arg); + (void)data; // unused + + size_t *num_all_blocks = arg; + (*num_all_blocks)++; +} + +static void ravl_cb_count_free(void *data, void *arg) { + assert(data); + assert(arg); + + ravl_data_t *node_data = data; + assert(node_data); + ravl_free_blocks_head_t *head_node = node_data->value; + assert(head_node); + struct ravl_free_blocks_elem_t *free_block = head_node->head; + assert(free_block); + + size_t *num_all_blocks = arg; + while (free_block) { + (*num_all_blocks)++; + free_block = free_block->next; + } +} + +static umf_result_t coarse_get_stats_no_lock(coarse_t *coarse, + coarse_stats_t *stats) { + assert(coarse); + + size_t num_all_blocks = 0; + ravl_foreach(coarse->all_blocks, ravl_cb_count, &num_all_blocks); + + size_t num_free_blocks = 0; + ravl_foreach(coarse->free_blocks, ravl_cb_count_free, &num_free_blocks); + + stats->alloc_size = coarse->alloc_size; + stats->used_size = coarse->used_size; + stats->num_all_blocks = num_all_blocks; + stats->num_free_blocks = num_free_blocks; + + return UMF_RESULT_SUCCESS; +} + +// PUBLIC API + +umf_result_t coarse_new(coarse_params_t *coarse_params, coarse_t **pcoarse) { +#ifdef _WIN32 + utils_init_once(&Log_initialized, utils_log_init); +#endif /* _WIN32 */ + + if (coarse_params == NULL || pcoarse == NULL) { + LOG_ERR("coarse parameters or handle is missing"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (!coarse_params->provider) { + LOG_ERR("memory provider is not set"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (!coarse_params->page_size) { + LOG_ERR("page size of the memory provider is not set"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (!coarse_params->cb.split) { + LOG_ERR("coarse split callback is not set"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (!coarse_params->cb.merge) { + LOG_ERR("coarse merge callback is not set"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + // alloc() and free() callbacks are optional + + coarse_t *coarse = umf_ba_global_alloc(sizeof(*coarse)); + if (!coarse) { + LOG_ERR("out of the host memory"); + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + memset(coarse, 0, sizeof(*coarse)); + + coarse->provider = coarse_params->provider; + coarse->page_size = coarse_params->page_size; + coarse->cb = coarse_params->cb; + coarse->allocation_strategy = coarse_params->allocation_strategy; + + umf_result_t umf_result = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + + coarse->free_blocks = ravl_new_sized(coarse_ravl_comp, sizeof(ravl_data_t)); + if (coarse->free_blocks == NULL) { + LOG_ERR("out of the host memory"); + goto err_free_coarse; + } + + coarse->all_blocks = ravl_new_sized(coarse_ravl_comp, sizeof(ravl_data_t)); + if (coarse->all_blocks == NULL) { + LOG_ERR("out of the host memory"); + goto err_delete_ravl_free_blocks; + } + + coarse->alloc_size = 0; + coarse->used_size = 0; + + umf_result = UMF_RESULT_ERROR_UNKNOWN; + + if (utils_mutex_init(&coarse->lock) == NULL) { + LOG_ERR("lock initialization failed"); + goto err_delete_ravl_all_blocks; + } + + assert(coarse->used_size == 0); + assert(coarse->alloc_size == 0); + assert(debug_check(coarse)); + + *pcoarse = coarse; + + return UMF_RESULT_SUCCESS; + +err_delete_ravl_all_blocks: + ravl_delete(coarse->all_blocks); +err_delete_ravl_free_blocks: + ravl_delete(coarse->free_blocks); +err_free_coarse: + umf_ba_global_free(coarse); + return umf_result; +} + +void coarse_delete(coarse_t *coarse) { + if (coarse == NULL) { + LOG_ERR("coarse handle is missing"); + return; + } + + utils_mutex_destroy_not_free(&coarse->lock); + + ravl_foreach(coarse->all_blocks, coarse_ravl_cb_rm_all_blocks_node, coarse); + assert(coarse->used_size == 0); + assert(coarse->alloc_size == 0); + + ravl_delete(coarse->all_blocks); + ravl_delete(coarse->free_blocks); + + umf_ba_global_free(coarse); +} + +umf_result_t coarse_add_memory_from_provider(coarse_t *coarse, size_t size) { + umf_result_t umf_result; + void *ptr = NULL; + + if (coarse == NULL || size == 0) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (!coarse->cb.alloc) { + LOG_ERR("error: alloc callback is not set"); + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + umf_result = coarse_alloc(coarse, size, coarse->page_size, &ptr); + if (umf_result != UMF_RESULT_SUCCESS) { + return umf_result; + } + + assert(ptr); + + return coarse_free(coarse, ptr, size); +} + +umf_result_t coarse_add_memory_fixed(coarse_t *coarse, void *addr, + size_t size) { + umf_result_t umf_result; + + if (coarse == NULL || addr == NULL || size == 0) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (coarse->cb.alloc || coarse->cb.free) { + LOG_ERR("error: alloc or free callback is set"); + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + if (utils_mutex_lock(&coarse->lock) != 0) { + LOG_ERR("locking the lock failed"); + return UMF_RESULT_ERROR_UNKNOWN; + } + + assert(debug_check(coarse)); + + umf_result = coarse_add_used_block(coarse, addr, size); + + assert(debug_check(coarse)); + utils_mutex_unlock(&coarse->lock); + + if (umf_result != UMF_RESULT_SUCCESS) { + return umf_result; + } + + umf_result = coarse_free(coarse, addr, size); + if (umf_result != UMF_RESULT_SUCCESS) { + return umf_result; + } + + LOG_DEBUG("coarse_ALLOC (add_memory_block) %zu used %zu alloc %zu", size, + coarse->used_size, coarse->alloc_size); + + return UMF_RESULT_SUCCESS; +} + +umf_result_t coarse_alloc(coarse_t *coarse, size_t size, size_t alignment, + void **resultPtr) { + umf_result_t umf_result = UMF_RESULT_ERROR_UNKNOWN; + + if (coarse == NULL || resultPtr == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + // alignment must be a power of two and a multiple or a divider of the page size + if (alignment == 0) { + alignment = coarse->page_size; + } else if ((alignment & (alignment - 1)) || + ((alignment % coarse->page_size) && + (coarse->page_size % alignment))) { + LOG_ERR("wrong alignment: %zu (not a power of 2 or a multiple or a " + "divider of the page size (%zu))", + alignment, coarse->page_size); + return UMF_RESULT_ERROR_INVALID_ALIGNMENT; + } else if (IS_NOT_ALIGNED(alignment, coarse->page_size)) { + alignment = ALIGN_UP_SAFE(alignment, coarse->page_size); + } + + if (utils_mutex_lock(&coarse->lock) != 0) { + LOG_ERR("locking the lock failed"); + return UMF_RESULT_ERROR_UNKNOWN; + } + + assert(debug_check(coarse)); + + // Find a block with greater or equal size using the given memory allocation strategy + block_t *curr = find_free_block(coarse->free_blocks, size, alignment, + coarse->allocation_strategy); + + // If the block that we want to reuse has a greater size, split it. + // Try to merge the split part with the successor if it is not used. + enum { ACTION_NONE = 0, ACTION_USE, ACTION_SPLIT } action = ACTION_NONE; + + if (curr && curr->size > size) { + action = ACTION_SPLIT; + } else if (curr && curr->size == size) { + action = ACTION_USE; + } + + if (action) { // ACTION_SPLIT or ACTION_USE + assert(curr->used == false); + + // In case of non-zero alignment create an aligned block what would be further used. + if (alignment > 0) { + umf_result = create_aligned_block(coarse, size, alignment, &curr); + if (umf_result != UMF_RESULT_SUCCESS) { + (void)free_blocks_re_add(coarse, curr); + goto err_unlock; + } + } + + if (action == ACTION_SPLIT) { + // Split the current block and put the new block after the one that we use. + umf_result = split_current_block(coarse, curr, size); + if (umf_result != UMF_RESULT_SUCCESS) { + (void)free_blocks_re_add(coarse, curr); + goto err_unlock; + } + + curr->size = size; + + LOG_DEBUG("coarse_ALLOC (split_block) %zu used %zu alloc %zu", size, + coarse->used_size, coarse->alloc_size); + + } else { // action == ACTION_USE + LOG_DEBUG("coarse_ALLOC (same_block) %zu used %zu alloc %zu", size, + coarse->used_size, coarse->alloc_size); + } + + curr->used = true; + *resultPtr = curr->data; + coarse->used_size += size; + + assert(debug_check(coarse)); + utils_mutex_unlock(&coarse->lock); + + return UMF_RESULT_SUCCESS; + } + + // no suitable block found - try to get more memory from the upstream provider + umf_result = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + + *resultPtr = NULL; + + if (!coarse->cb.alloc) { + LOG_ERR("out of memory"); + goto err_unlock; + } + + umf_result = coarse->cb.alloc(coarse->provider, size, alignment, resultPtr); + if (umf_result != UMF_RESULT_SUCCESS) { + LOG_ERR("coarse_alloc_cb() failed: out of memory"); + goto err_unlock; + } + + ASSERT_IS_ALIGNED(((uintptr_t)(*resultPtr)), alignment); + + umf_result = coarse_add_used_block(coarse, *resultPtr, size); + if (umf_result != UMF_RESULT_SUCCESS) { + if (coarse->cb.free) { + coarse->cb.free(coarse->provider, *resultPtr, size); + } + goto err_unlock; + } + + LOG_DEBUG("coarse_ALLOC (memory_provider) %zu used %zu alloc %zu", size, + coarse->used_size, coarse->alloc_size); + + umf_result = UMF_RESULT_SUCCESS; + +err_unlock: + assert(debug_check(coarse)); + utils_mutex_unlock(&coarse->lock); + + return umf_result; +} + +umf_result_t coarse_free(coarse_t *coarse, void *ptr, size_t bytes) { + if (coarse == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (ptr == NULL) { + return UMF_RESULT_SUCCESS; + } + + if (utils_mutex_lock(&coarse->lock) != 0) { + LOG_ERR("locking the lock failed"); + return UMF_RESULT_ERROR_UNKNOWN; + } + + assert(debug_check(coarse)); + + ravl_node_t *node = coarse_ravl_find_node(coarse->all_blocks, ptr); + if (node == NULL) { + // the block was not found + LOG_ERR("memory block not found (ptr = %p, size = %zu)", ptr, bytes); + utils_mutex_unlock(&coarse->lock); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + block_t *block = get_node_block(node); + assert(block->used); + + if (bytes > 0 && bytes != block->size) { + // wrong size of allocation + LOG_ERR("wrong size of allocation"); + utils_mutex_unlock(&coarse->lock); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + LOG_DEBUG("coarse_FREE (return_block_to_pool) %zu used %zu alloc %zu", + block->size, coarse->used_size - block->size, coarse->alloc_size); + + assert(coarse->used_size >= block->size); + coarse->used_size -= block->size; + + block->used = false; + + // Merge with prev and/or next block if they are unused and have continuous data. + node = free_block_merge_with_prev(coarse, node); + node = free_block_merge_with_next(coarse, node); + + int rv = free_blocks_add(coarse->free_blocks, get_node_block(node)); + if (rv) { + utils_mutex_unlock(&coarse->lock); + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + assert(debug_check(coarse)); + utils_mutex_unlock(&coarse->lock); + + return UMF_RESULT_SUCCESS; +} + +umf_result_t coarse_merge(coarse_t *coarse, void *lowPtr, void *highPtr, + size_t totalSize) { + umf_result_t umf_result; + + if (coarse == NULL || lowPtr == NULL || highPtr == NULL || totalSize == 0 || + ((uintptr_t)highPtr <= (uintptr_t)lowPtr) || + ((uintptr_t)highPtr - (uintptr_t)lowPtr >= totalSize)) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (utils_mutex_lock(&coarse->lock) != 0) { + LOG_ERR("locking the lock failed"); + return UMF_RESULT_ERROR_UNKNOWN; + } + + assert(debug_check(coarse)); + + umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; + + ravl_node_t *low_node = coarse_ravl_find_node(coarse->all_blocks, lowPtr); + if (low_node == NULL) { + LOG_ERR("the lowPtr memory block not found"); + goto err_mutex_unlock; + } + + block_t *low_block = get_node_block(low_node); + if (!low_block->used) { + LOG_ERR("the lowPtr block is not allocated"); + goto err_mutex_unlock; + } + + ravl_node_t *high_node = coarse_ravl_find_node(coarse->all_blocks, highPtr); + if (high_node == NULL) { + LOG_ERR("the highPtr memory block not found"); + goto err_mutex_unlock; + } + + block_t *high_block = get_node_block(high_node); + if (!high_block->used) { + LOG_ERR("the highPtr block is not allocated"); + goto err_mutex_unlock; + } + + if (get_node_next(low_node) != high_node || + ((uintptr_t)highPtr != ((uintptr_t)lowPtr + low_block->size))) { + LOG_ERR("given allocations are not adjacent"); + goto err_mutex_unlock; + } + + assert(get_node_prev(high_node) == low_node); + + if (low_block->size + high_block->size != totalSize) { + LOG_ERR("wrong totalSize: %zu != %zu", totalSize, + low_block->size + high_block->size); + goto err_mutex_unlock; + } + + ravl_node_t *merged_node = NULL; + + umf_result = + user_block_merge(coarse, low_node, high_node, true, &merged_node); + if (umf_result != UMF_RESULT_SUCCESS) { + LOG_ERR("merging a block failed"); + goto err_mutex_unlock; + } + + assert(merged_node == low_node); + assert(low_block->size == totalSize); + + umf_result = UMF_RESULT_SUCCESS; + +err_mutex_unlock: + assert(debug_check(coarse)); + utils_mutex_unlock(&coarse->lock); + + return umf_result; +} + +umf_result_t coarse_split(coarse_t *coarse, void *ptr, size_t totalSize, + size_t firstSize) { + umf_result_t umf_result; + + if (coarse == NULL || ptr == NULL || (firstSize >= totalSize) || + firstSize == 0 || totalSize == 0) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (utils_mutex_lock(&coarse->lock) != 0) { + LOG_ERR("locking the lock failed"); + return UMF_RESULT_ERROR_UNKNOWN; + } + + assert(debug_check(coarse)); + + umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; + + ravl_node_t *node = coarse_ravl_find_node(coarse->all_blocks, ptr); + if (node == NULL) { + LOG_ERR("memory block not found"); + goto err_mutex_unlock; + } + + block_t *block = get_node_block(node); + + if (block->size != totalSize) { + LOG_ERR("wrong totalSize: %zu != %zu", totalSize, block->size); + goto err_mutex_unlock; + } + + if (!block->used) { + LOG_ERR("block is not allocated"); + goto err_mutex_unlock; + } + + // check if block can be split by the memory provider + umf_result = can_provider_split(coarse, ptr, totalSize, firstSize); + if (umf_result != UMF_RESULT_SUCCESS) { + LOG_ERR("memory provider cannot split a memory block"); + goto err_mutex_unlock; + } + + umf_result = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + + block_t *new_block = + coarse_ravl_add_new(coarse->all_blocks, block->data + firstSize, + block->size - firstSize, NULL); + if (new_block == NULL) { + goto err_mutex_unlock; + } + + block->size = firstSize; + new_block->used = true; + + assert(new_block->size == (totalSize - firstSize)); + + umf_result = UMF_RESULT_SUCCESS; + +err_mutex_unlock: + assert(debug_check(coarse)); + utils_mutex_unlock(&coarse->lock); + + return umf_result; +} + +coarse_stats_t coarse_get_stats(coarse_t *coarse) { + coarse_stats_t stats = {0}; + + if (coarse == NULL) { + return stats; + } + + if (utils_mutex_lock(&coarse->lock) != 0) { + LOG_ERR("locking the lock failed"); + return stats; + } + + coarse_get_stats_no_lock(coarse, &stats); + + utils_mutex_unlock(&coarse->lock); + + return stats; +} diff --git a/src/coarse/coarse.h b/src/coarse/coarse.h new file mode 100644 index 0000000000..93ec990027 --- /dev/null +++ b/src/coarse/coarse.h @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#ifndef UMF_COARSE_H +#define UMF_COARSE_H + +#include +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct coarse_t coarse_t; + +// coarse callbacks implement provider-specific actions +typedef struct coarse_callbacks_t { + // alloc() is optional (can be NULL for the fixed-size memory provider) + umf_result_t (*alloc)(void *provider, size_t size, size_t alignment, + void **ptr); + // free() is optional (can be NULL if the provider does not support the free() op) + umf_result_t (*free)(void *provider, void *ptr, size_t size); + umf_result_t (*split)(void *provider, void *ptr, size_t totalSize, + size_t firstSize); + umf_result_t (*merge)(void *provider, void *lowPtr, void *highPtr, + size_t totalSize); +} coarse_callbacks_t; + +// coarse library allocation strategy +typedef enum coarse_strategy_t { + // Check if the first free block of the 'size' size has the correct alignment. + // If not, use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. + UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE = 0, + + // Always allocate a free block of the (size + alignment) size + // and cut out the properly aligned part leaving two remaining parts. + // It is the fastest strategy but causes memory fragmentation + // when alignment is greater than 0. + // It is the best strategy when alignment always equals 0. + UMF_COARSE_MEMORY_STRATEGY_FASTEST, + + // Look through all free blocks of the 'size' size + // and choose the first one with the correct alignment. + // If none of them had the correct alignment, + // use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. + UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE, +} coarse_strategy_t; + +// coarse library settings structure +typedef struct coarse_params_t { + // handle of the memory provider + void *provider; + + // coarse callbacks + coarse_callbacks_t cb; + + // memory allocation strategy, + // see coarse_strategy_t for details + coarse_strategy_t allocation_strategy; + + // page size of the memory provider + size_t page_size; +} coarse_params_t; + +// coarse library statistics +typedef struct coarse_stats_t { + // total allocation size + size_t alloc_size; + + // size of used memory + size_t used_size; + + // total number of allocated memory blocks + size_t num_all_blocks; + + // number of free memory blocks + size_t num_free_blocks; +} coarse_stats_t; + +umf_result_t coarse_new(coarse_params_t *coarse_params, coarse_t **pcoarse); +void coarse_delete(coarse_t *coarse); + +umf_result_t coarse_alloc(coarse_t *coarse, size_t size, size_t alignment, + void **resultPtr); +umf_result_t coarse_free(coarse_t *coarse, void *ptr, size_t bytes); + +umf_result_t coarse_merge(coarse_t *coarse, void *lowPtr, void *highPtr, + size_t totalSize); +umf_result_t coarse_split(coarse_t *coarse, void *ptr, size_t totalSize, + size_t firstSize); + +// supported only if the alloc callback is set, +// returns UMF_RESULT_ERROR_NOT_SUPPORTED otherwise +umf_result_t coarse_add_memory_from_provider(coarse_t *coarse, size_t size); + +// supported only if the alloc and the free callbacks are NOT set +// returns UMF_RESULT_ERROR_NOT_SUPPORTED otherwise +umf_result_t coarse_add_memory_fixed(coarse_t *coarse, void *addr, size_t size); + +coarse_stats_t coarse_get_stats(coarse_t *coarse); + +#ifdef __cplusplus +} +#endif + +#endif // UMF_COARSE_H diff --git a/src/cpp_helpers.hpp b/src/cpp_helpers.hpp index 6316ccbc7e..8789105814 100644 --- a/src/cpp_helpers.hpp +++ b/src/cpp_helpers.hpp @@ -84,7 +84,7 @@ template constexpr umf_memory_provider_ops_t providerOpsBase() { ops.version = UMF_VERSION_CURRENT; ops.finalize = [](void *obj) { delete reinterpret_cast(obj); }; UMF_ASSIGN_OP(ops, T, alloc, UMF_RESULT_ERROR_UNKNOWN); - UMF_ASSIGN_OP(ops.ext, T, free, UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP(ops, T, free, UMF_RESULT_ERROR_UNKNOWN); UMF_ASSIGN_OP_NORETURN(ops, T, get_last_native_error); UMF_ASSIGN_OP(ops, T, get_recommended_page_size, UMF_RESULT_ERROR_UNKNOWN); UMF_ASSIGN_OP(ops, T, get_min_page_size, UMF_RESULT_ERROR_UNKNOWN); diff --git a/src/ctl/ctl.c b/src/ctl/ctl.c new file mode 100644 index 0000000000..4db11ac21f --- /dev/null +++ b/src/ctl/ctl.c @@ -0,0 +1,596 @@ +/* + * + * Copyright (C) 2016-2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +// This file was originally under following license: +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright 2024, Intel Corporation */ + +/* + * ctl.c -- implementation of the interface for examination and modification of + * the library's internal state + */ + +#include "ctl.h" + +#include +#include +#include +#include +#include + +#include "base_alloc/base_alloc_global.h" +#include "utils/utils_common.h" +#include "utlist.h" + +#ifdef _WIN32 +#define strtok_r strtok_s +#else +#include +#endif + +#define CTL_MAX_ENTRIES 100 + +#define MAX_CONFIG_FILE_LEN (1 << 20) /* 1 megabyte */ + +#define CTL_STRING_QUERY_SEPARATOR ";" +#define CTL_NAME_VALUE_SEPARATOR "=" +#define CTL_QUERY_NODE_SEPARATOR "." +#define CTL_VALUE_ARG_SEPARATOR "," + +static int ctl_global_first_free = 0; +static struct ctl_node CTL_NODE(global)[CTL_MAX_ENTRIES]; + +/* + * This is the top level node of the ctl tree structure. Each node can contain + * children and leaf nodes. + * + * Internal nodes simply create a new path in the tree whereas child nodes are + * the ones providing the read/write functionality by the means of callbacks. + * + * Each tree node must be NULL-terminated, CTL_NODE_END macro is provided for + * convenience. + */ +struct ctl { + struct ctl_node root[CTL_MAX_ENTRIES]; + int first_free; +}; + +void *Zalloc(size_t sz) { + void *ptr = umf_ba_global_alloc(sz); + if (ptr) { + memset(ptr, 0, sz); + } + return ptr; +} + +char *Strdup(const char *s) { + size_t len = strlen(s) + 1; + char *p = umf_ba_global_alloc(len); + if (p) { + memcpy(p, s, len); + } + return p; +} + +/* + * ctl_find_node -- (internal) searches for a matching entry point in the + * provided nodes + * + * The caller is responsible for freeing all of the allocated indexes, + * regardless of the return value. + */ +static const struct ctl_node *ctl_find_node(const struct ctl_node *nodes, + const char *name, + struct ctl_index_utlist *indexes) { + const struct ctl_node *n = NULL; + char *sptr = NULL; + char *parse_str = Strdup(name); + if (parse_str == NULL) { + return NULL; + } + + char *node_name = strtok_r(parse_str, CTL_QUERY_NODE_SEPARATOR, &sptr); + + /* + * Go through the string and separate tokens that correspond to nodes + * in the main ctl tree. + */ + while (node_name != NULL) { + char *endptr; + /* + * Ignore errno from strtol: FreeBSD returns EINVAL if no + * conversion is performed. Linux does not, but endptr + * check is valid in both cases. + */ + int tmp_errno = errno; + long index_value = strtol(node_name, &endptr, 0); + errno = tmp_errno; + struct ctl_index_utlist *index_entry = NULL; + if (endptr != node_name) { /* a valid index */ + index_entry = umf_ba_global_alloc(sizeof(*index_entry)); + if (index_entry == NULL) { + goto error; + } + index_entry->value = index_value; + LL_PREPEND(indexes, index_entry); + } + + for (n = &nodes[0]; n->name != NULL; ++n) { + if (index_entry && n->type == CTL_NODE_INDEXED) { + break; + } else if (strcmp(n->name, node_name) == 0) { + break; + } + } + if (n->name == NULL) { + goto error; + } + + if (index_entry) { + index_entry->name = n->name; + } + + nodes = n->children; + node_name = strtok_r(NULL, CTL_QUERY_NODE_SEPARATOR, &sptr); + } + + umf_ba_global_free(parse_str); + return n; + +error: + umf_ba_global_free(parse_str); + return NULL; +} + +/* + * ctl_delete_indexes -- + * (internal) removes and frees all entries on the index list + */ +static void ctl_delete_indexes(struct ctl_index_utlist *indexes) { + if (!indexes) { + return; + } + struct ctl_index_utlist *elem, *tmp; + LL_FOREACH_SAFE(indexes, elem, tmp) { + LL_DELETE(indexes, elem); + if (elem) { + umf_ba_global_free(elem); + } + } +} + +/* + * ctl_parse_args -- (internal) parses a string argument based on the node + * structure + */ +static void *ctl_parse_args(const struct ctl_argument *arg_proto, char *arg) { + char *dest_arg = umf_ba_global_alloc(arg_proto->dest_size); + if (dest_arg == NULL) { + return NULL; + } + + char *sptr = NULL; + char *arg_sep = strtok_r(arg, CTL_VALUE_ARG_SEPARATOR, &sptr); + for (const struct ctl_argument_parser *p = arg_proto->parsers; + p->parser != NULL; ++p) { + if (arg_sep == NULL) { + goto error_parsing; + } + + if (p->parser(arg_sep, dest_arg + p->dest_offset, p->dest_size) != 0) { + goto error_parsing; + } + + arg_sep = strtok_r(NULL, CTL_VALUE_ARG_SEPARATOR, &sptr); + } + + return dest_arg; + +error_parsing: + umf_ba_global_free(dest_arg); + return NULL; +} + +/* + * ctl_query_get_real_args -- (internal) returns a pointer with actual argument + * structure as required by the node callback + */ +static void *ctl_query_get_real_args(const struct ctl_node *n, void *write_arg, + enum ctl_query_source source) { + void *real_arg = NULL; + switch (source) { + case CTL_QUERY_CONFIG_INPUT: + real_arg = ctl_parse_args(n->arg, write_arg); + break; + case CTL_QUERY_PROGRAMMATIC: + real_arg = write_arg; + break; + default: + break; + } + + return real_arg; +} + +/* + * ctl_query_cleanup_real_args -- (internal) cleanups relevant argument + * structures allocated as a result of the get_real_args call + */ +static void ctl_query_cleanup_real_args(const struct ctl_node *n, + void *real_arg, + enum ctl_query_source source) { + /* suppress unused-parameter errors */ + (void)n; + + switch (source) { + case CTL_QUERY_CONFIG_INPUT: + umf_ba_global_free(real_arg); + break; + case CTL_QUERY_PROGRAMMATIC: + break; + default: + break; + } +} + +/* + * ctl_exec_query_read -- (internal) calls the read callback of a node + */ +static int ctl_exec_query_read(void *ctx, const struct ctl_node *n, + enum ctl_query_source source, void *arg, + struct ctl_index_utlist *indexes) { + if (arg == NULL) { + errno = EINVAL; + return -1; + } + + return n->cb[CTL_QUERY_READ](ctx, source, arg, indexes); +} + +/* + * ctl_exec_query_write -- (internal) calls the write callback of a node + */ +static int ctl_exec_query_write(void *ctx, const struct ctl_node *n, + enum ctl_query_source source, void *arg, + struct ctl_index_utlist *indexes) { + if (arg == NULL) { + errno = EINVAL; + return -1; + } + + void *real_arg = ctl_query_get_real_args(n, arg, source); + if (real_arg == NULL) { + return -1; + } + + int ret = n->cb[CTL_QUERY_WRITE](ctx, source, real_arg, indexes); + ctl_query_cleanup_real_args(n, real_arg, source); + + return ret; +} + +/* + * ctl_exec_query_runnable -- (internal) calls the run callback of a node + */ +static int ctl_exec_query_runnable(void *ctx, const struct ctl_node *n, + enum ctl_query_source source, void *arg, + struct ctl_index_utlist *indexes) { + return n->cb[CTL_QUERY_RUNNABLE](ctx, source, arg, indexes); +} + +static int (*ctl_exec_query[MAX_CTL_QUERY_TYPE])( + void *ctx, const struct ctl_node *n, enum ctl_query_source source, + void *arg, struct ctl_index_utlist *indexes) = { + ctl_exec_query_read, + ctl_exec_query_write, + ctl_exec_query_runnable, +}; + +/* + * ctl_query -- (internal) parses the name and calls the appropriate methods + * from the ctl tree + */ +int ctl_query(struct ctl *ctl, void *ctx, enum ctl_query_source source, + const char *name, enum ctl_query_type type, void *arg) { + if (name == NULL) { + errno = EINVAL; + return -1; + } + + /* + * All of the indexes are put on this list so that the handlers can + * easily retrieve the index values. The list is cleared once the ctl + * query has been handled. + */ + struct ctl_index_utlist *indexes = NULL; + indexes = Zalloc(sizeof(*indexes)); + if (!indexes) { + return -1; + } + + int ret = -1; + + const struct ctl_node *n = ctl_find_node(CTL_NODE(global), name, indexes); + + if (n == NULL && ctl) { + ctl_delete_indexes(indexes); + indexes = NULL; + n = ctl_find_node(ctl->root, name, indexes); + } + + if (n == NULL || n->type != CTL_NODE_LEAF || n->cb[type] == NULL) { + errno = EINVAL; + goto out; + } + + ret = ctl_exec_query[type](ctx, n, source, arg, indexes); + +out: + ctl_delete_indexes(indexes); + + return ret; +} + +/* + * ctl_register_module_node -- adds a new node to the CTL tree root. + */ +void ctl_register_module_node(struct ctl *c, const char *name, + struct ctl_node *n) { + struct ctl_node *nnode = c == NULL + ? &CTL_NODE(global)[ctl_global_first_free++] + : &c->root[c->first_free++]; + + nnode->children = n; + nnode->type = CTL_NODE_NAMED; + nnode->name = name; +} + +/* + * ctl_parse_query -- (internal) splits an entire query string + * into name and value + */ +static int ctl_parse_query(char *qbuf, char **name, char **value) { + if (qbuf == NULL) { + return -1; + } + + char *sptr = NULL; + *name = strtok_r(qbuf, CTL_NAME_VALUE_SEPARATOR, &sptr); + if (*name == NULL) { + return -1; + } + + *value = strtok_r(NULL, CTL_NAME_VALUE_SEPARATOR, &sptr); + if (*value == NULL) { + return -1; + } + + /* the value itself mustn't include CTL_NAME_VALUE_SEPARATOR */ + char *extra = strtok_r(NULL, CTL_NAME_VALUE_SEPARATOR, &sptr); + if (extra != NULL) { + return -1; + } + + return 0; +} + +/* + * ctl_load_config -- executes the entire query collection from a provider + */ +static int ctl_load_config(struct ctl *ctl, void *ctx, char *buf) { + int r = 0; + char *sptr = NULL; /* for internal use of strtok */ + char *name; + char *value; + char *qbuf = strtok_r(buf, CTL_STRING_QUERY_SEPARATOR, &sptr); + + while (qbuf != NULL) { + r = ctl_parse_query(qbuf, &name, &value); + if (r != 0) { + return -1; + } + + r = ctl_query(ctl, ctx, CTL_QUERY_CONFIG_INPUT, name, CTL_QUERY_WRITE, + value); + + if (r < 0 && ctx != NULL) { + return -1; + } + + qbuf = strtok_r(NULL, CTL_STRING_QUERY_SEPARATOR, &sptr); + } + + return 0; +} + +/* + * ctl_load_config_from_string -- loads obj configuration from string + */ +int ctl_load_config_from_string(struct ctl *ctl, void *ctx, + const char *cfg_string) { + char *buf = Strdup(cfg_string); + if (buf == NULL) { + return -1; + } + + int ret = ctl_load_config(ctl, ctx, buf); + + umf_ba_global_free(buf); + return ret; +} + +/* + * ctl_load_config_from_file -- loads obj configuration from file + * + * This function opens up the config file, allocates a buffer of size equal to + * the size of the file, reads its content and sanitizes it for ctl_load_config. + */ +#ifndef _WIN32 // TODO: implement for Windows +int ctl_load_config_from_file(struct ctl *ctl, void *ctx, + const char *cfg_file) { + int ret = -1; + long fsize = 0; + char *buf = NULL; + + FILE *fp = fopen(cfg_file, "r"); + if (fp == NULL) { + return ret; + } + + int err; + if ((err = fseek(fp, 0, SEEK_END)) != 0) { + goto error_file_parse; + } + + fsize = ftell(fp); + if (fsize == -1) { + goto error_file_parse; + } + + if (fsize > MAX_CONFIG_FILE_LEN) { + goto error_file_parse; + } + + if ((err = fseek(fp, 0, SEEK_SET)) != 0) { + goto error_file_parse; + } + + buf = Zalloc((size_t)fsize + 1); /* +1 for NULL-termination */ + if (buf == NULL) { + goto error_file_parse; + } + + { + size_t bufpos = 0; + int c; + int is_comment_section = 0; + while ((c = fgetc(fp)) != EOF) { + if (c == '#') { + is_comment_section = 1; + } else if (c == '\n') { + is_comment_section = 0; + } else if (!is_comment_section && !isspace(c)) { + buf[bufpos++] = (char)c; + } + } + } + + ret = ctl_load_config(ctl, ctx, buf); + + umf_ba_global_free(buf); + +error_file_parse: + (void)fclose(fp); + return ret; +} +#endif + +/* + * ctl_new -- allocates and initializes ctl data structures + */ +struct ctl *ctl_new(void) { + struct ctl *c = Zalloc(sizeof(struct ctl)); + if (c == NULL) { + return NULL; + } + + c->first_free = 0; + return c; +} + +/* + * ctl_delete -- deletes ctl + */ +void ctl_delete(struct ctl *c) { umf_ba_global_free(c); } + +/* + * ctl_parse_ll -- (internal) parses and returns a long long signed integer + */ +static long long ctl_parse_ll(const char *str) { + char *endptr; + int olderrno = errno; + errno = 0; + long long val = strtoll(str, &endptr, 0); + if (endptr == str || errno != 0) { + return LLONG_MIN; + } + errno = olderrno; + + return val; +} + +/* + * ctl_arg_boolean -- checks whether the provided argument contains + * either a 1 or y or Y. + */ +int ctl_arg_boolean(const void *arg, void *dest, size_t dest_size) { + /* suppress unused-parameter errors */ + (void)dest_size; + + int *intp = dest; + char in = ((const char *)arg)[0]; + + if (tolower(in) == 'y' || in == '1') { + *intp = 1; + return 0; + } else if (tolower(in) == 'n' || in == '0') { + *intp = 0; + return 0; + } + + return -1; +} + +/* + * ctl_arg_integer -- parses signed integer argument + */ +int ctl_arg_integer(const void *arg, void *dest, size_t dest_size) { + long long val = ctl_parse_ll(arg); + if (val == LLONG_MIN) { + return -1; + } + + switch (dest_size) { + case sizeof(int): + if (val > INT_MAX || val < INT_MIN) { + return -1; + } + *(int *)dest = (int)val; + break; + case sizeof(long long): + *(long long *)dest = val; + break; + case sizeof(uint8_t): + if (val > UINT8_MAX || val < 0) { + return -1; + } + *(uint8_t *)dest = (uint8_t)val; + break; + default: + errno = EINVAL; + return -1; + } + + return 0; +} + +/* + * ctl_arg_string -- verifies length and copies a string argument into a zeroed + * buffer + */ +int ctl_arg_string(const void *arg, void *dest, size_t dest_size) { + /* check if the incoming string is longer or equal to dest_size */ + if (strnlen(arg, dest_size) == dest_size) { + return -1; + } + + strncpy(dest, arg, dest_size); + + return 0; +} diff --git a/src/ctl/ctl.h b/src/ctl/ctl.h new file mode 100644 index 0000000000..9327b01afe --- /dev/null +++ b/src/ctl/ctl.h @@ -0,0 +1,216 @@ +/* + * + * Copyright (C) 2016-2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +// This file was originally under following license: +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright 2016-2020, Intel Corporation */ + +/* + * ctl.h -- internal declaration of statistics and control related structures + */ + +#ifndef UMF_CTL_H +#define UMF_CTL_H 1 + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct ctl; + +struct ctl_index_utlist { + const char *name; + long value; + struct ctl_index_utlist *next; +}; + +enum ctl_query_source { + CTL_UNKNOWN_QUERY_SOURCE, + /* query executed directly from the program */ + CTL_QUERY_PROGRAMMATIC, + /* query executed from the config file */ + CTL_QUERY_CONFIG_INPUT, + + MAX_CTL_QUERY_SOURCE +}; + +enum ctl_query_type { + CTL_QUERY_READ, + CTL_QUERY_WRITE, + CTL_QUERY_RUNNABLE, + + MAX_CTL_QUERY_TYPE +}; + +typedef int (*node_callback)(void *ctx, enum ctl_query_source type, void *arg, + struct ctl_index_utlist *indexes); + +enum ctl_node_type { + CTL_NODE_UNKNOWN, + CTL_NODE_NAMED, + CTL_NODE_LEAF, + CTL_NODE_INDEXED, + + MAX_CTL_NODE +}; + +typedef int (*ctl_arg_parser)(const void *arg, void *dest, size_t dest_size); + +struct ctl_argument_parser { + size_t dest_offset; /* offset of the field inside of the argument */ + size_t dest_size; /* size of the field inside of the argument */ + ctl_arg_parser parser; +}; + +struct ctl_argument { + size_t dest_size; /* size of the entire argument */ + struct ctl_argument_parser parsers[]; /* array of 'fields' in arg */ +}; + +#define sizeof_member(t, m) sizeof(((t *)0)->m) + +#define CTL_ARG_PARSER(t, p) \ + { 0, sizeof(t), p } + +#define CTL_ARG_PARSER_STRUCT(t, m, p) \ + { offsetof(t, m), sizeof_member(t, m), p } + +#define CTL_ARG_PARSER_END \ + { 0, 0, NULL } + +/* + * CTL Tree node structure, do not use directly. All the necessary functionality + * is provided by the included macros. + */ +struct ctl_node { + const char *name; + enum ctl_node_type type; + + node_callback cb[MAX_CTL_QUERY_TYPE]; + const struct ctl_argument *arg; + + const struct ctl_node *children; +}; + +struct ctl *ctl_new(void); +void ctl_delete(struct ctl *stats); + +int ctl_load_config_from_string(struct ctl *ctl, void *ctx, + const char *cfg_string); +int ctl_load_config_from_file(struct ctl *ctl, void *ctx, const char *cfg_file); + +/* Use through CTL_REGISTER_MODULE, never directly */ +void ctl_register_module_node(struct ctl *c, const char *name, + struct ctl_node *n); + +int ctl_arg_boolean(const void *arg, void *dest, size_t dest_size); +#define CTL_ARG_BOOLEAN \ + {sizeof(int), {{0, sizeof(int), ctl_arg_boolean}, CTL_ARG_PARSER_END}}; + +int ctl_arg_integer(const void *arg, void *dest, size_t dest_size); +#define CTL_ARG_INT \ + {sizeof(int), {{0, sizeof(int), ctl_arg_integer}, CTL_ARG_PARSER_END}}; + +#define CTL_ARG_LONG_LONG \ + { \ + sizeof(long long), { \ + {0, sizeof(long long), ctl_arg_integer}, CTL_ARG_PARSER_END \ + } \ + } + +int ctl_arg_string(const void *arg, void *dest, size_t dest_size); +#define CTL_ARG_STRING(len) \ + {len, {{0, len, ctl_arg_string}, CTL_ARG_PARSER_END}}; + +#define CTL_STR(name) #name + +#define CTL_NODE_END \ + { NULL, CTL_NODE_UNKNOWN, {NULL, NULL, NULL}, NULL, NULL } + +#define CTL_NODE(name, ...) ctl_node_##__VA_ARGS__##_##name + +int ctl_query(struct ctl *ctl, void *ctx, enum ctl_query_source source, + const char *name, enum ctl_query_type type, void *arg); + +/* Declaration of a new child node */ +#define CTL_CHILD(name, ...) \ + { \ + CTL_STR(name), CTL_NODE_NAMED, {NULL, NULL, NULL}, NULL, \ + (struct ctl_node *)CTL_NODE(name, __VA_ARGS__) \ + } + +/* Declaration of a new indexed node */ +#define CTL_INDEXED(name, ...) \ + { \ + CTL_STR(name), CTL_NODE_INDEXED, {NULL, NULL, NULL}, NULL, \ + (struct ctl_node *)CTL_NODE(name, __VA_ARGS__) \ + } + +#define CTL_READ_HANDLER(name, ...) ctl_##__VA_ARGS__##_##name##_read + +#define CTL_WRITE_HANDLER(name, ...) ctl_##__VA_ARGS__##_##name##_write + +#define CTL_RUNNABLE_HANDLER(name, ...) ctl_##__VA_ARGS__##_##name##_runnable + +#define CTL_ARG(name) ctl_arg_##name + +/* + * Declaration of a new read-only leaf. If used the corresponding read function + * must be declared by CTL_READ_HANDLER macro. + */ +#define CTL_LEAF_RO(name, ...) \ + { \ + CTL_STR(name), CTL_NODE_LEAF, \ + {CTL_READ_HANDLER(name, __VA_ARGS__), NULL, NULL}, NULL, NULL \ + } + +/* + * Declaration of a new write-only leaf. If used the corresponding write + * function must be declared by CTL_WRITE_HANDLER macro. + */ +#define CTL_LEAF_WO(name, ...) \ + { \ + CTL_STR(name), CTL_NODE_LEAF, \ + {NULL, CTL_WRITE_HANDLER(name, __VA_ARGS__), NULL}, \ + &CTL_ARG(name), NULL \ + } + +/* + * Declaration of a new runnable leaf. If used the corresponding run + * function must be declared by CTL_RUNNABLE_HANDLER macro. + */ +#define CTL_LEAF_RUNNABLE(name, ...) \ + { \ + CTL_STR(name), CTL_NODE_LEAF, \ + {NULL, NULL, CTL_RUNNABLE_HANDLER(name, __VA_ARGS__)}, NULL, NULL \ + } + +/* + * Declaration of a new read-write leaf. If used both read and write function + * must be declared by CTL_READ_HANDLER and CTL_WRITE_HANDLER macros. + */ +#define CTL_LEAF_RW(name) \ + { \ + CTL_STR(name), CTL_NODE_LEAF, \ + {CTL_READ_HANDLER(name), CTL_WRITE_HANDLER(name), NULL}, \ + &CTL_ARG(name), NULL \ + } + +#define CTL_REGISTER_MODULE(_ctl, name) \ + ctl_register_module_node((_ctl), CTL_STR(name), \ + (struct ctl_node *)CTL_NODE(name)) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/libumf.def b/src/libumf.def index 82e32d4a13..42d7cfaf3c 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -1,12 +1,12 @@ ;;;; Begin Copyright Notice -; Copyright (C) 2024 Intel Corporation +; Copyright (C) 2023-2025 Intel Corporation ; Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. ; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ;;;; End Copyright Notice LIBRARY UMF -VERSION 0.10 +VERSION 0.11 EXPORTS DllMain @@ -14,8 +14,6 @@ EXPORTS umfTearDown umfGetCurrentVersion umfCloseIPCHandle - umfCoarseMemoryProviderGetStats - umfCoarseMemoryProviderOps umfCUDAMemoryProviderOps umfCUDAMemoryProviderParamsCreate umfCUDAMemoryProviderParamsDestroy @@ -27,15 +25,19 @@ EXPORTS umfDevDaxMemoryProviderParamsDestroy umfDevDaxMemoryProviderParamsSetDeviceDax umfDevDaxMemoryProviderParamsSetProtection - umfFree umfFileMemoryProviderOps umfFileMemoryProviderParamsCreate umfFileMemoryProviderParamsDestroy umfFileMemoryProviderParamsSetPath umfFileMemoryProviderParamsSetProtection umfFileMemoryProviderParamsSetVisibility + umfFixedMemoryProviderOps + umfFixedMemoryProviderParamsCreate + umfFixedMemoryProviderParamsDestroy + umfFree umfGetIPCHandle umfGetLastFailedMemoryProvider + umfJemallocPoolOps umfLevelZeroMemoryProviderOps umfLevelZeroMemoryProviderParamsCreate umfLevelZeroMemoryProviderParamsDestroy @@ -105,10 +107,12 @@ EXPORTS umfPoolGetIPCHandler umfPoolGetIPCHandleSize umfPoolGetLastAllocationError + umfPoolGetTag umfPoolGetMemoryProvider umfPoolMalloc umfPoolMallocUsableSize umfPoolRealloc + umfPoolSetTag umfProxyPoolOps umfPutIPCHandle umfScalablePoolOps @@ -116,3 +120,5 @@ EXPORTS umfScalablePoolParamsDestroy umfScalablePoolParamsSetGranularity umfScalablePoolParamsSetKeepAllMemory +; Added in UMF_0.11 + umfLevelZeroMemoryProviderParamsSetFreePolicy diff --git a/src/libumf.map b/src/libumf.map index 4755b6b814..c33bb7c10f 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -1,4 +1,4 @@ -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2023-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -8,8 +8,6 @@ UMF_0.10 { umfTearDown; umfGetCurrentVersion; umfCloseIPCHandle; - umfCoarseMemoryProviderGetStats; - umfCoarseMemoryProviderOps; umfCUDAMemoryProviderOps; umfCUDAMemoryProviderParamsCreate; umfCUDAMemoryProviderParamsDestroy; @@ -21,15 +19,16 @@ UMF_0.10 { umfDevDaxMemoryProviderParamsDestroy; umfDevDaxMemoryProviderParamsSetDeviceDax; umfDevDaxMemoryProviderParamsSetProtection; - umfFree; umfFileMemoryProviderOps; umfFileMemoryProviderParamsCreate; umfFileMemoryProviderParamsDestroy; umfFileMemoryProviderParamsSetPath; umfFileMemoryProviderParamsSetProtection; umfFileMemoryProviderParamsSetVisibility; + umfFree; umfGetIPCHandle; umfGetLastFailedMemoryProvider; + umfJemallocPoolOps; umfLevelZeroMemoryProviderOps; umfLevelZeroMemoryProviderParamsCreate; umfLevelZeroMemoryProviderParamsDestroy; @@ -82,13 +81,13 @@ UMF_0.10 { umfOsMemoryProviderOps; umfOsMemoryProviderParamsCreate; umfOsMemoryProviderParamsDestroy; - umfOsMemoryProviderParamsSetProtection; - umfOsMemoryProviderParamsSetVisibility; - umfOsMemoryProviderParamsSetShmName; umfOsMemoryProviderParamsSetNumaList; umfOsMemoryProviderParamsSetNumaMode; umfOsMemoryProviderParamsSetPartSize; umfOsMemoryProviderParamsSetPartitions; + umfOsMemoryProviderParamsSetProtection; + umfOsMemoryProviderParamsSetShmName; + umfOsMemoryProviderParamsSetVisibility; umfPoolAlignedMalloc; umfPoolByPtr; umfPoolCalloc; @@ -100,9 +99,11 @@ UMF_0.10 { umfPoolGetIPCHandleSize; umfPoolGetLastAllocationError; umfPoolGetMemoryProvider; + umfPoolGetTag; umfPoolMalloc; umfPoolMallocUsableSize; umfPoolRealloc; + umfPoolSetTag; umfProxyPoolOps; umfPutIPCHandle; umfScalablePoolOps; @@ -113,3 +114,10 @@ UMF_0.10 { local: *; }; + +UMF_0.11 { + umfFixedMemoryProviderOps; + umfFixedMemoryProviderParamsCreate; + umfFixedMemoryProviderParamsDestroy; + umfLevelZeroMemoryProviderParamsSetFreePolicy; +} UMF_0.10; diff --git a/src/libumf.rc.in b/src/libumf.rc.in index 7aba79e7ed..8ee85d6268 100644 --- a/src/libumf.rc.in +++ b/src/libumf.rc.in @@ -1,4 +1,4 @@ -// Copyright (c) 2024 Intel Corporation +// Copyright (C) 2024-2025 Intel Corporation // // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -51,7 +51,7 @@ BEGIN VALUE "CompanyName", "Intel Corporation\0" VALUE "FileDescription", "Unified Memory Framework (UMF) library\0" VALUE "FileVersion", _UMF_VERSION "\0" - VALUE "LegalCopyright", "Copyright 2024, Intel Corporation. All rights reserved.\0" + VALUE "LegalCopyright", "Copyright 2024-2025, Intel Corporation. All rights reserved.\0" VALUE "LegalTrademarks", "\0" VALUE "OriginalFilename", "umf.dll\0" VALUE "ProductName", "Unified Memory Framework (UMF)\0" diff --git a/src/memory_pool.c b/src/memory_pool.c index 4a85955efa..e739f3f2fc 100644 --- a/src/memory_pool.c +++ b/src/memory_pool.c @@ -42,10 +42,7 @@ static umf_result_t umfPoolCreateInternal(const umf_memory_pool_ops_t *ops, if (!(flags & UMF_POOL_CREATE_FLAG_DISABLE_TRACKING)) { // Wrap provider with memory tracking provider. - // Check if the provider supports the free() operation. - bool upstreamDoesNotFree = umfIsFreeOpDefault(provider); - ret = umfTrackingMemoryProviderCreate(provider, pool, &pool->provider, - upstreamDoesNotFree); + ret = umfTrackingMemoryProviderCreate(provider, pool, &pool->provider); if (ret != UMF_RESULT_SUCCESS) { goto err_provider_create; } @@ -55,6 +52,13 @@ static umf_result_t umfPoolCreateInternal(const umf_memory_pool_ops_t *ops, pool->flags = flags; pool->ops = *ops; + pool->tag = NULL; + + if (NULL == utils_mutex_init(&pool->lock)) { + LOG_ERR("Failed to initialize mutex for pool"); + ret = UMF_RESULT_ERROR_UNKNOWN; + goto err_lock_init; + } ret = ops->initialize(pool->provider, params, &pool->pool_priv); if (ret != UMF_RESULT_SUCCESS) { @@ -66,6 +70,8 @@ static umf_result_t umfPoolCreateInternal(const umf_memory_pool_ops_t *ops, return UMF_RESULT_SUCCESS; err_pool_init: + utils_mutex_destroy_not_free(&pool->lock); +err_lock_init: if (!(flags & UMF_POOL_CREATE_FLAG_DISABLE_TRACKING)) { umfMemoryProviderDestroy(pool->provider); } @@ -90,6 +96,8 @@ void umfPoolDestroy(umf_memory_pool_handle_t hPool) { umfMemoryProviderDestroy(hUpstreamProvider); } + utils_mutex_destroy_not_free(&hPool->lock); + LOG_INFO("Memory pool destroyed: %p", (void *)hPool); // TODO: this free keeps memory in base allocator, so it can lead to OOM in some scenarios (it should be optimized) @@ -175,3 +183,24 @@ umf_result_t umfPoolGetLastAllocationError(umf_memory_pool_handle_t hPool) { UMF_CHECK((hPool != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); return hPool->ops.get_last_allocation_error(hPool->pool_priv); } + +umf_result_t umfPoolSetTag(umf_memory_pool_handle_t hPool, void *tag, + void **oldTag) { + UMF_CHECK((hPool != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + utils_mutex_lock(&hPool->lock); + if (oldTag) { + *oldTag = hPool->tag; + } + hPool->tag = tag; + utils_mutex_unlock(&hPool->lock); + return UMF_RESULT_SUCCESS; +} + +umf_result_t umfPoolGetTag(umf_memory_pool_handle_t hPool, void **tag) { + UMF_CHECK((hPool != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK((tag != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + utils_mutex_lock(&hPool->lock); + *tag = hPool->tag; + utils_mutex_unlock(&hPool->lock); + return UMF_RESULT_SUCCESS; +} diff --git a/src/memory_pool_internal.h b/src/memory_pool_internal.h index 90f2f16298..ab3378163d 100644 --- a/src/memory_pool_internal.h +++ b/src/memory_pool_internal.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -22,6 +22,7 @@ extern "C" { #endif #include "base_alloc.h" +#include "utils_concurrency.h" typedef struct umf_memory_pool_t { void *pool_priv; @@ -30,6 +31,9 @@ typedef struct umf_memory_pool_t { // Memory provider used by the pool. umf_memory_provider_handle_t provider; + + utils_mutex_t lock; + void *tag; } umf_memory_pool_t; #ifdef __cplusplus diff --git a/src/memory_provider.c b/src/memory_provider.c index 883f1be263..59f3f12592 100644 --- a/src/memory_provider.c +++ b/src/memory_provider.c @@ -25,13 +25,6 @@ typedef struct umf_memory_provider_t { void *provider_priv; } umf_memory_provider_t; -static umf_result_t umfDefaultFree(void *provider, void *ptr, size_t size) { - (void)provider; - (void)ptr; - (void)size; - return UMF_RESULT_ERROR_NOT_SUPPORTED; -} - static umf_result_t umfDefaultPurgeLazy(void *provider, void *ptr, size_t size) { (void)provider; @@ -106,9 +99,6 @@ static umf_result_t umfDefaultCloseIPCHandle(void *provider, void *ptr, } void assignOpsExtDefaults(umf_memory_provider_ops_t *ops) { - if (!ops->ext.free) { - ops->ext.free = umfDefaultFree; - } if (!ops->ext.purge_lazy) { ops->ext.purge_lazy = umfDefaultPurgeLazy; } @@ -143,7 +133,7 @@ void assignOpsIpcDefaults(umf_memory_provider_ops_t *ops) { static bool validateOpsMandatory(const umf_memory_provider_ops_t *ops) { // Mandatory ops should be non-NULL - return ops->alloc && ops->get_recommended_page_size && + return ops->alloc && ops->free && ops->get_recommended_page_size && ops->get_min_page_size && ops->initialize && ops->finalize && ops->get_last_native_error && ops->get_name; } @@ -169,10 +159,6 @@ static bool validateOps(const umf_memory_provider_ops_t *ops) { validateOpsIpc(&(ops->ipc)); } -bool umfIsFreeOpDefault(umf_memory_provider_handle_t hProvider) { - return (hProvider->ops.ext.free == umfDefaultFree); -} - umf_result_t umfMemoryProviderCreate(const umf_memory_provider_ops_t *ops, void *params, umf_memory_provider_handle_t *hProvider) { @@ -236,8 +222,7 @@ umf_result_t umfMemoryProviderAlloc(umf_memory_provider_handle_t hProvider, umf_result_t umfMemoryProviderFree(umf_memory_provider_handle_t hProvider, void *ptr, size_t size) { UMF_CHECK((hProvider != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); - umf_result_t res = - hProvider->ops.ext.free(hProvider->provider_priv, ptr, size); + umf_result_t res = hProvider->ops.free(hProvider->provider_priv, ptr, size); checkErrorAndSetLastProvider(res, hProvider); return res; } diff --git a/src/memory_provider_get_last_failed.c b/src/memory_provider_get_last_failed.c index 9434eea976..09bd075e10 100644 --- a/src/memory_provider_get_last_failed.c +++ b/src/memory_provider_get_last_failed.c @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/memory_provider_internal.h b/src/memory_provider_internal.h index 49b2f2e531..0b7f45f80d 100644 --- a/src/memory_provider_internal.h +++ b/src/memory_provider_internal.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -20,7 +20,6 @@ extern "C" { void *umfMemoryProviderGetPriv(umf_memory_provider_handle_t hProvider); umf_memory_provider_handle_t *umfGetLastFailedMemoryProviderPtr(void); -bool umfIsFreeOpDefault(umf_memory_provider_handle_t hProvider); #ifdef __cplusplus } diff --git a/src/memspaces/memspace_numa.c b/src/memspaces/memspace_numa.c index 0028e394dc..83e65fc291 100644 --- a/src/memspaces/memspace_numa.c +++ b/src/memspaces/memspace_numa.c @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/memtargets/memtarget_numa.h b/src/memtargets/memtarget_numa.h index 2d3e3fd704..6659d045ef 100644 --- a/src/memtargets/memtarget_numa.h +++ b/src/memtargets/memtarget_numa.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/pool/CMakeLists.txt b/src/pool/CMakeLists.txt index bdd196b041..f54e701857 100644 --- a/src/pool/CMakeLists.txt +++ b/src/pool/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (C) 2023 Intel Corporation +# Copyright (C) 2023-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -38,17 +38,3 @@ if(UMF_BUILD_LIBUMF_POOL_DISJOINT) install(TARGETS disjoint_pool EXPORT ${PROJECT_NAME}-targets) endif() - -# libumf_pool_jemalloc -if(UMF_BUILD_LIBUMF_POOL_JEMALLOC) - add_umf_library( - NAME jemalloc_pool - TYPE STATIC - SRCS pool_jemalloc.c ${POOL_EXTRA_SRCS} - LIBS jemalloc ${POOL_EXTRA_LIBS}) - target_include_directories(jemalloc_pool PRIVATE ${JEMALLOC_INCLUDE_DIRS}) - target_compile_definitions(jemalloc_pool - PRIVATE ${POOL_COMPILE_DEFINITIONS}) - add_library(${PROJECT_NAME}::jemalloc_pool ALIAS jemalloc_pool) - install(TARGETS jemalloc_pool EXPORT ${PROJECT_NAME}-targets) -endif() diff --git a/src/pool/pool_disjoint.cpp b/src/pool/pool_disjoint.cpp index e0298b43df..0390f53751 100644 --- a/src/pool/pool_disjoint.cpp +++ b/src/pool/pool_disjoint.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/pool/pool_jemalloc.c b/src/pool/pool_jemalloc.c index 3ec7c78050..88e8e93422 100644 --- a/src/pool/pool_jemalloc.c +++ b/src/pool/pool_jemalloc.c @@ -20,33 +20,21 @@ #include #include -#include +#ifndef UMF_POOL_JEMALLOC_ENABLED -// The Windows version of jemalloc uses API with je_ prefix, -// while the Linux one does not. -#ifndef _WIN32 -#define je_mallocx mallocx -#define je_dallocx dallocx -#define je_rallocx rallocx -#define je_mallctl mallctl -#define je_malloc_usable_size malloc_usable_size -#endif +umf_memory_pool_ops_t *umfJemallocPoolOps(void) { return NULL; } + +#else + +#include #define MALLOCX_ARENA_MAX (MALLCTL_ARENAS_ALL - 1) typedef struct jemalloc_memory_pool_t { umf_memory_provider_handle_t provider; unsigned int arena_index; // index of jemalloc arena - // set to true if umfMemoryProviderFree() should never be called - bool disable_provider_free; } jemalloc_memory_pool_t; -// Configuration of Jemalloc Pool -typedef struct umf_jemalloc_pool_params_t { - /// Set to true if umfMemoryProviderFree() should never be called. - bool disable_provider_free; -} umf_jemalloc_pool_params_t; - static __TLS umf_result_t TLS_last_allocation_error; static jemalloc_memory_pool_t *pool_by_arena_index[MALLCTL_ARENAS_ALL]; @@ -59,52 +47,6 @@ static jemalloc_memory_pool_t *get_pool_by_arena_index(unsigned arena_ind) { return pool_by_arena_index[arena_ind]; } -umf_result_t -umfJemallocPoolParamsCreate(umf_jemalloc_pool_params_handle_t *hParams) { - if (!hParams) { - LOG_ERR("jemalloc pool params handle is NULL"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - umf_jemalloc_pool_params_t *params_data = - umf_ba_global_alloc(sizeof(*params_data)); - if (!params_data) { - LOG_ERR("cannot allocate memory for jemalloc poolparams"); - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - params_data->disable_provider_free = false; - - *hParams = (umf_jemalloc_pool_params_handle_t)params_data; - - return UMF_RESULT_SUCCESS; -} - -umf_result_t -umfJemallocPoolParamsDestroy(umf_jemalloc_pool_params_handle_t hParams) { - if (!hParams) { - LOG_ERR("jemalloc pool params handle is NULL"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - umf_ba_global_free(hParams); - - return UMF_RESULT_SUCCESS; -} - -umf_result_t -umfJemallocPoolParamsSetKeepAllMemory(umf_jemalloc_pool_params_handle_t hParams, - bool keepAllMemory) { - if (!hParams) { - LOG_ERR("jemalloc pool params handle is NULL"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - hParams->disable_provider_free = keepAllMemory; - - return UMF_RESULT_SUCCESS; -} - // arena_extent_alloc - an extent allocation function conforms to the extent_alloc_t type and upon // success returns a pointer to size bytes of mapped memory on behalf of arena arena_ind such that // the extent's base address is a multiple of alignment, as well as setting *zero to indicate @@ -134,9 +76,7 @@ static void *arena_extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, } if (new_addr != NULL && ptr != new_addr) { - if (!pool->disable_provider_free) { - umfMemoryProviderFree(pool->provider, ptr, size); - } + umfMemoryProviderFree(pool->provider, ptr, size); return NULL; } @@ -170,10 +110,6 @@ static void arena_extent_destroy(extent_hooks_t *extent_hooks, void *addr, jemalloc_memory_pool_t *pool = get_pool_by_arena_index(arena_ind); - if (pool->disable_provider_free) { - return; - } - umf_result_t ret; ret = umfMemoryProviderFree(pool->provider, addr, size); if (ret != UMF_RESULT_SUCCESS) { @@ -196,10 +132,6 @@ static bool arena_extent_dalloc(extent_hooks_t *extent_hooks, void *addr, jemalloc_memory_pool_t *pool = get_pool_by_arena_index(arena_ind); - if (pool->disable_provider_free) { - return true; // opt-out from deallocation - } - umf_result_t ret; ret = umfMemoryProviderFree(pool->provider, addr, size); if (ret != UMF_RESULT_SUCCESS) { @@ -450,12 +382,10 @@ static void *op_aligned_alloc(void *pool, size_t size, size_t alignment) { static umf_result_t op_initialize(umf_memory_provider_handle_t provider, void *params, void **out_pool) { + (void)params; // unused assert(provider); assert(out_pool); - umf_jemalloc_pool_params_handle_t je_params = - (umf_jemalloc_pool_params_handle_t)params; - extent_hooks_t *pHooks = &arena_extent_hooks; size_t unsigned_size = sizeof(unsigned); int err; @@ -468,12 +398,6 @@ static umf_result_t op_initialize(umf_memory_provider_handle_t provider, pool->provider = provider; - if (je_params) { - pool->disable_provider_free = je_params->disable_provider_free; - } else { - pool->disable_provider_free = false; - } - unsigned arena_index; err = je_mallctl("arenas.create", (void *)&arena_index, &unsigned_size, NULL, 0); @@ -488,7 +412,7 @@ static umf_result_t op_initialize(umf_memory_provider_handle_t provider, err = je_mallctl(cmd, NULL, NULL, (void *)&pHooks, sizeof(void *)); if (err) { snprintf(cmd, sizeof(cmd), "arena.%u.destroy", arena_index); - je_mallctl(cmd, NULL, 0, NULL, 0); + (void)je_mallctl(cmd, NULL, 0, NULL, 0); LOG_ERR("Could not setup extent_hooks for newly created arena."); goto err_free_pool; } @@ -512,7 +436,7 @@ static void op_finalize(void *pool) { jemalloc_memory_pool_t *je_pool = (jemalloc_memory_pool_t *)pool; char cmd[64]; snprintf(cmd, sizeof(cmd), "arena.%u.destroy", je_pool->arena_index); - je_mallctl(cmd, NULL, 0, NULL, 0); + (void)je_mallctl(cmd, NULL, 0, NULL, 0); pool_by_arena_index[je_pool->arena_index] = NULL; umf_ba_global_free(je_pool); @@ -545,3 +469,4 @@ static umf_memory_pool_ops_t UMF_JEMALLOC_POOL_OPS = { umf_memory_pool_ops_t *umfJemallocPoolOps(void) { return &UMF_JEMALLOC_POOL_OPS; } +#endif /* UMF_POOL_JEMALLOC_ENABLED */ diff --git a/src/provider/provider_coarse.c b/src/provider/provider_coarse.c deleted file mode 100644 index c3027b91d7..0000000000 --- a/src/provider/provider_coarse.c +++ /dev/null @@ -1,1707 +0,0 @@ -/* - * Copyright (C) 2023-2024 Intel Corporation - * - * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -*/ - -#include -#include -#include -#include -#include -#include - -#include - -#include "base_alloc_global.h" -#include "memory_provider_internal.h" -#include "ravl.h" -#include "utils_common.h" -#include "utils_concurrency.h" -#include "utils_log.h" - -#define COARSE_BASE_NAME "coarse" - -#define IS_ORIGIN_OF_BLOCK(origin, block) \ - (((uintptr_t)(block)->data >= (uintptr_t)(origin)->data) && \ - ((uintptr_t)(block)->data + (block)->size <= \ - (uintptr_t)(origin)->data + (origin)->size)) - -typedef struct coarse_memory_provider_t { - umf_memory_provider_handle_t upstream_memory_provider; - - // destroy upstream_memory_provider in finalize() - bool destroy_upstream_memory_provider; - - // memory allocation strategy - coarse_memory_provider_strategy_t allocation_strategy; - - void *init_buffer; - - size_t used_size; - size_t alloc_size; - - // upstream_blocks - tree of all blocks allocated from the upstream provider - struct ravl *upstream_blocks; - - // all_blocks - tree of all blocks - sorted by an address of data - struct ravl *all_blocks; - - // free_blocks - tree of free blocks - sorted by a size of data, - // each node contains a pointer (ravl_free_blocks_head_t) - // to the head of the list of free blocks of the same size - struct ravl *free_blocks; - - struct utils_mutex_t lock; - - // Name of the provider with the upstream provider: - // "coarse ()" - // for example: "coarse (L0)" - char *name; - - // Set to true if the free() operation of the upstream memory provider is not supported - // (i.e. if (umfMemoryProviderFree(upstream_memory_provider, NULL, 0) == UMF_RESULT_ERROR_NOT_SUPPORTED) - bool disable_upstream_provider_free; -} coarse_memory_provider_t; - -typedef struct ravl_node ravl_node_t; - -typedef enum check_free_blocks_t { - CHECK_ONLY_THE_FIRST_BLOCK = 0, - CHECK_ALL_BLOCKS_OF_SIZE, -} check_free_blocks_t; - -typedef struct block_t { - size_t size; - unsigned char *data; - bool used; - - // Node in the list of free blocks of the same size pointing to this block. - // The list is located in the (coarse_provider->free_blocks) RAVL tree. - struct ravl_free_blocks_elem_t *free_list_ptr; -} block_t; - -// A general node in a RAVL tree. -// 1) coarse_provider->all_blocks RAVL tree (tree of all blocks - sorted by an address of data): -// key - pointer (block_t->data) to the beginning of the block data -// value - pointer (block_t) to the block of the allocation -// 2) coarse_provider->free_blocks RAVL tree (tree of free blocks - sorted by a size of data): -// key - size of the allocation (block_t->size) -// value - pointer (ravl_free_blocks_head_t) to the head of the list of free blocks of the same size -typedef struct ravl_data_t { - uintptr_t key; - void *value; -} ravl_data_t; - -// The head of the list of free blocks of the same size. -typedef struct ravl_free_blocks_head_t { - struct ravl_free_blocks_elem_t *head; -} ravl_free_blocks_head_t; - -// The node of the list of free blocks of the same size -typedef struct ravl_free_blocks_elem_t { - struct block_t *block; - struct ravl_free_blocks_elem_t *next; - struct ravl_free_blocks_elem_t *prev; -} ravl_free_blocks_elem_t; - -// The compare function of a RAVL tree -static int coarse_ravl_comp(const void *lhs, const void *rhs) { - const ravl_data_t *lhs_ravl = (const ravl_data_t *)lhs; - const ravl_data_t *rhs_ravl = (const ravl_data_t *)rhs; - - if (lhs_ravl->key < rhs_ravl->key) { - return -1; - } - - if (lhs_ravl->key > rhs_ravl->key) { - return 1; - } - - // lhs_ravl->key == rhs_ravl->key - return 0; -} - -static inline block_t *get_node_block(ravl_node_t *node) { - ravl_data_t *node_data = ravl_data(node); - assert(node_data); - assert(node_data->value); - return node_data->value; -} - -static inline ravl_node_t *get_node_prev(ravl_node_t *node) { - return ravl_node_predecessor(node); -} - -static inline ravl_node_t *get_node_next(ravl_node_t *node) { - return ravl_node_successor(node); -} - -#ifndef NDEBUG -static block_t *get_block_prev(ravl_node_t *node) { - ravl_node_t *ravl_prev = ravl_node_predecessor(node); - if (!ravl_prev) { - return NULL; - } - - return get_node_block(ravl_prev); -} - -static block_t *get_block_next(ravl_node_t *node) { - ravl_node_t *ravl_next = ravl_node_successor(node); - if (!ravl_next) { - return NULL; - } - - return get_node_block(ravl_next); -} -#endif /* NDEBUG */ - -static bool is_same_origin(struct ravl *upstream_blocks, block_t *block1, - block_t *block2) { - ravl_data_t rdata1 = {(uintptr_t)block1->data, NULL}; - ravl_node_t *ravl_origin1 = - ravl_find(upstream_blocks, &rdata1, RAVL_PREDICATE_LESS_EQUAL); - assert(ravl_origin1); - - block_t *origin1 = get_node_block(ravl_origin1); - assert(IS_ORIGIN_OF_BLOCK(origin1, block1)); - - return (IS_ORIGIN_OF_BLOCK(origin1, block2)); -} - -// The functions "coarse_ravl_*" handle lists of blocks: -// - coarse_provider->all_blocks and coarse_provider->upstream_blocks -// sorted by a pointer (block_t->data) to the beginning of the block data. -// -// coarse_ravl_add_new - allocate and add a new block to the tree -// and link this block to the next and the previous one. -static block_t *coarse_ravl_add_new(struct ravl *rtree, unsigned char *data, - size_t size, ravl_node_t **node) { - assert(rtree); - assert(data); - assert(size); - - // TODO add valgrind annotations - block_t *block = umf_ba_global_alloc(sizeof(*block)); - if (block == NULL) { - return NULL; - } - - block->data = data; - block->size = size; - block->free_list_ptr = NULL; - - ravl_data_t rdata = {(uintptr_t)block->data, block}; - assert(NULL == ravl_find(rtree, &data, RAVL_PREDICATE_EQUAL)); - int ret = ravl_emplace_copy(rtree, &rdata); - if (ret) { - umf_ba_global_free(block); - return NULL; - } - - ravl_node_t *new_node = ravl_find(rtree, &rdata, RAVL_PREDICATE_EQUAL); - assert(NULL != new_node); - - if (node) { - *node = new_node; - } - - return block; -} - -// coarse_ravl_find_node - find the node in the tree -static ravl_node_t *coarse_ravl_find_node(struct ravl *rtree, void *ptr) { - ravl_data_t data = {(uintptr_t)ptr, NULL}; - return ravl_find(rtree, &data, RAVL_PREDICATE_EQUAL); -} - -// coarse_ravl_rm - remove the block from the tree -static block_t *coarse_ravl_rm(struct ravl *rtree, void *ptr) { - ravl_data_t data = {(uintptr_t)ptr, NULL}; - ravl_node_t *node; - node = ravl_find(rtree, &data, RAVL_PREDICATE_EQUAL); - if (node) { - ravl_data_t *node_data = ravl_data(node); - assert(node_data); - block_t *block = node_data->value; - assert(block); - ravl_remove(rtree, node); - assert(NULL == ravl_find(rtree, &data, RAVL_PREDICATE_EQUAL)); - return block; - } - return NULL; -} - -// The functions "node_list_*" handle lists of free blocks of the same size. -// The heads (ravl_free_blocks_head_t) of those lists are stored in nodes of -// the coarse_provider->free_blocks RAVL tree. -// -// node_list_add - add a free block to the list of free blocks of the same size -static ravl_free_blocks_elem_t * -node_list_add(ravl_free_blocks_head_t *head_node, struct block_t *block) { - assert(head_node); - assert(block); - - ravl_free_blocks_elem_t *node = umf_ba_global_alloc(sizeof(*node)); - if (node == NULL) { - return NULL; - } - - if (head_node->head) { - head_node->head->prev = node; - } - - node->block = block; - node->next = head_node->head; - node->prev = NULL; - head_node->head = node; - - return node; -} - -// node_list_rm - remove the given free block from the list of free blocks of the same size -static block_t *node_list_rm(ravl_free_blocks_head_t *head_node, - ravl_free_blocks_elem_t *node) { - assert(head_node); - assert(node); - - if (!head_node->head) { - return NULL; - } - - if (node == head_node->head) { - assert(node->prev == NULL); - head_node->head = node->next; - } - - ravl_free_blocks_elem_t *node_next = node->next; - ravl_free_blocks_elem_t *node_prev = node->prev; - if (node_next) { - node_next->prev = node_prev; - } - - if (node_prev) { - node_prev->next = node_next; - } - - struct block_t *block = node->block; - block->free_list_ptr = NULL; - umf_ba_global_free(node); - - return block; -} - -// node_list_rm_first - remove the first free block from the list of free blocks of the same size only if it can be properly aligned -static block_t *node_list_rm_first(ravl_free_blocks_head_t *head_node, - size_t alignment) { - assert(head_node); - - if (!head_node->head) { - return NULL; - } - - ravl_free_blocks_elem_t *node = head_node->head; - assert(node->prev == NULL); - struct block_t *block = node->block; - - if (IS_NOT_ALIGNED(block->size, alignment)) { - return NULL; - } - - if (node->next) { - node->next->prev = NULL; - } - - head_node->head = node->next; - block->free_list_ptr = NULL; - umf_ba_global_free(node); - - return block; -} - -// node_list_rm_with_alignment - remove the first free block with the correct alignment from the list of free blocks of the same size -static block_t *node_list_rm_with_alignment(ravl_free_blocks_head_t *head_node, - size_t alignment) { - assert(head_node); - - if (!head_node->head) { - return NULL; - } - - assert(((ravl_free_blocks_elem_t *)head_node->head)->prev == NULL); - - ravl_free_blocks_elem_t *node; - for (node = head_node->head; node != NULL; node = node->next) { - if (IS_ALIGNED(node->block->size, alignment)) { - return node_list_rm(head_node, node); - } - } - - return NULL; -} - -// The functions "free_blocks_*" handle the coarse_provider->free_blocks RAVL tree -// sorted by a size of the allocation (block_t->size). -// This is a tree of heads (ravl_free_blocks_head_t) of lists of free blocks of the same size. -// -// free_blocks_add - add a free block to the list of free blocks of the same size -static int free_blocks_add(struct ravl *free_blocks, block_t *block) { - ravl_free_blocks_head_t *head_node = NULL; - int rv; - - ravl_data_t head_node_data = {(uintptr_t)block->size, NULL}; - ravl_node_t *node; - node = ravl_find(free_blocks, &head_node_data, RAVL_PREDICATE_EQUAL); - if (node) { - ravl_data_t *node_data = ravl_data(node); - assert(node_data); - head_node = node_data->value; - assert(head_node); - } else { // no head_node - head_node = umf_ba_global_alloc(sizeof(*head_node)); - if (!head_node) { - return -1; - } - - head_node->head = NULL; - - ravl_data_t data = {(uintptr_t)block->size, head_node}; - rv = ravl_emplace_copy(free_blocks, &data); - if (rv) { - umf_ba_global_free(head_node); - return -1; - } - } - - block->free_list_ptr = node_list_add(head_node, block); - if (!block->free_list_ptr) { - return -1; - } - - assert(block->free_list_ptr->block->size == block->size); - - return 0; -} - -// free_blocks_rm_ge - remove the first free block of a size greater or equal to the given size only if it can be properly aligned -// If it was the last block, the head node is freed and removed from the tree. -// It is used during memory allocation (looking for a free block). -static block_t *free_blocks_rm_ge(struct ravl *free_blocks, size_t size, - size_t alignment, - check_free_blocks_t check_blocks) { - ravl_data_t data = {(uintptr_t)size, NULL}; - ravl_node_t *node; - node = ravl_find(free_blocks, &data, RAVL_PREDICATE_GREATER_EQUAL); - if (!node) { - return NULL; - } - - ravl_data_t *node_data = ravl_data(node); - assert(node_data); - assert(node_data->key >= size); - - ravl_free_blocks_head_t *head_node = node_data->value; - assert(head_node); - - block_t *block; - switch (check_blocks) { - case CHECK_ONLY_THE_FIRST_BLOCK: - block = node_list_rm_first(head_node, alignment); - break; - case CHECK_ALL_BLOCKS_OF_SIZE: - block = node_list_rm_with_alignment(head_node, alignment); - break; - // wrong value of check_blocks - default: - abort(); - } - - if (head_node->head == NULL) { - umf_ba_global_free(head_node); - ravl_remove(free_blocks, node); - } - - return block; -} - -// free_blocks_rm_node - remove the free block pointed by the given node. -// If it was the last block, the head node is freed and removed from the tree. -// It is used during merging free blocks and destroying the coarse_provider->free_blocks tree. -static block_t *free_blocks_rm_node(struct ravl *free_blocks, - ravl_free_blocks_elem_t *node) { - assert(free_blocks); - assert(node); - size_t size = node->block->size; - ravl_data_t data = {(uintptr_t)size, NULL}; - ravl_node_t *ravl_node; - ravl_node = ravl_find(free_blocks, &data, RAVL_PREDICATE_EQUAL); - assert(ravl_node); - - ravl_data_t *node_data = ravl_data(ravl_node); - assert(node_data); - assert(node_data->key == size); - - ravl_free_blocks_head_t *head_node = node_data->value; - assert(head_node); - - block_t *block = node_list_rm(head_node, node); - - if (head_node->head == NULL) { - umf_ba_global_free(head_node); - ravl_remove(free_blocks, ravl_node); - } - - return block; -} - -// user_block_merge - merge two blocks from one of two lists of user blocks: all_blocks or free_blocks -static umf_result_t user_block_merge(coarse_memory_provider_t *coarse_provider, - ravl_node_t *node1, ravl_node_t *node2, - bool used, ravl_node_t **merged_node) { - assert(node1); - assert(node2); - assert(node1 == get_node_prev(node2)); - assert(node2 == get_node_next(node1)); - assert(merged_node); - - *merged_node = NULL; - - struct ravl *upstream_blocks = coarse_provider->upstream_blocks; - struct ravl *all_blocks = coarse_provider->all_blocks; - struct ravl *free_blocks = coarse_provider->free_blocks; - - block_t *block1 = get_node_block(node1); - block_t *block2 = get_node_block(node2); - assert(block1->data < block2->data); - - bool same_used = ((block1->used == used) && (block2->used == used)); - bool contignous_data = (block1->data + block1->size == block2->data); - bool same_origin = is_same_origin(upstream_blocks, block1, block2); - - // check if blocks can be merged - if (!same_used || !contignous_data || !same_origin) { - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - if (block1->free_list_ptr) { - free_blocks_rm_node(free_blocks, block1->free_list_ptr); - block1->free_list_ptr = NULL; - } - - if (block2->free_list_ptr) { - free_blocks_rm_node(free_blocks, block2->free_list_ptr); - block2->free_list_ptr = NULL; - } - - // update the size - block1->size += block2->size; - - block_t *block_rm = coarse_ravl_rm(all_blocks, block2->data); - assert(block_rm == block2); - (void)block_rm; // WA for unused variable error - umf_ba_global_free(block2); - - *merged_node = node1; - - return UMF_RESULT_SUCCESS; -} - -// free_block_merge_with_prev - merge the given free block -// with the previous one if both are unused and have continuous data. -// Remove the merged block from the tree of free blocks. -static ravl_node_t * -free_block_merge_with_prev(coarse_memory_provider_t *coarse_provider, - ravl_node_t *node) { - ravl_node_t *node_prev = get_node_prev(node); - if (!node_prev) { - return node; - } - - ravl_node_t *merged_node = NULL; - umf_result_t umf_result = - user_block_merge(coarse_provider, node_prev, node, false, &merged_node); - if (umf_result != UMF_RESULT_SUCCESS) { - return node; - } - - assert(merged_node != NULL); - - return merged_node; -} - -// free_block_merge_with_next - merge the given free block -// with the next one if both are unused and have continuous data. -// Remove the merged block from the tree of free blocks. -static ravl_node_t * -free_block_merge_with_next(coarse_memory_provider_t *coarse_provider, - ravl_node_t *node) { - ravl_node_t *node_next = get_node_next(node); - if (!node_next) { - return node; - } - - ravl_node_t *merged_node = NULL; - umf_result_t umf_result = - user_block_merge(coarse_provider, node, node_next, false, &merged_node); - if (umf_result != UMF_RESULT_SUCCESS) { - return node; - } - - assert(merged_node != NULL); - - return merged_node; -} - -// upstream_block_merge - merge the given two upstream blocks -static umf_result_t -upstream_block_merge(coarse_memory_provider_t *coarse_provider, - ravl_node_t *node1, ravl_node_t *node2, - ravl_node_t **merged_node) { - assert(node1); - assert(node2); - assert(merged_node); - - *merged_node = NULL; - - umf_memory_provider_handle_t upstream_provider = - coarse_provider->upstream_memory_provider; - if (!upstream_provider) { - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - block_t *block1 = get_node_block(node1); - block_t *block2 = get_node_block(node2); - assert(block1->data < block2->data); - - bool contignous_data = (block1->data + block1->size == block2->data); - if (!contignous_data) { - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - // check if blocks can be merged by the upstream provider - umf_result_t merge_status = umfMemoryProviderAllocationMerge( - coarse_provider->upstream_memory_provider, block1->data, block2->data, - block1->size + block2->size); - if (merge_status != UMF_RESULT_SUCCESS) { - return merge_status; - } - - // update the size - block1->size += block2->size; - - struct ravl *upstream_blocks = coarse_provider->upstream_blocks; - block_t *block_rm = coarse_ravl_rm(upstream_blocks, block2->data); - assert(block_rm == block2); - (void)block_rm; // WA for unused variable error - umf_ba_global_free(block2); - - *merged_node = node1; - - return UMF_RESULT_SUCCESS; -} - -// upstream_block_merge_with_prev - merge the given upstream block -// with the previous one if both have continuous data. -// Remove the merged block from the tree of upstream blocks. -static ravl_node_t * -upstream_block_merge_with_prev(coarse_memory_provider_t *coarse_provider, - ravl_node_t *node) { - assert(node); - - ravl_node_t *node_prev = get_node_prev(node); - if (!node_prev) { - return node; - } - - ravl_node_t *merged_node = NULL; - umf_result_t umf_result = - upstream_block_merge(coarse_provider, node_prev, node, &merged_node); - if (umf_result != UMF_RESULT_SUCCESS) { - return node; - } - - assert(merged_node != NULL); - - return merged_node; -} - -// upstream_block_merge_with_next - merge the given upstream block -// with the next one if both have continuous data. -// Remove the merged block from the tree of upstream blocks. -static ravl_node_t * -upstream_block_merge_with_next(coarse_memory_provider_t *coarse_provider, - ravl_node_t *node) { - assert(node); - - ravl_node_t *node_next = get_node_next(node); - if (!node_next) { - return node; - } - - ravl_node_t *merged_node = NULL; - umf_result_t umf_result = - upstream_block_merge(coarse_provider, node, node_next, &merged_node); - if (umf_result != UMF_RESULT_SUCCESS) { - return node; - } - - assert(merged_node != NULL); - - return merged_node; -} - -#ifndef NDEBUG // begin of DEBUG code - -typedef struct debug_cb_args_t { - coarse_memory_provider_t *provider; - size_t sum_used; - size_t sum_blocks_size; - size_t num_all_blocks; - size_t num_free_blocks; - size_t num_alloc_blocks; - size_t sum_alloc_size; -} debug_cb_args_t; - -static void debug_verify_all_blocks_cb(void *data, void *arg) { - assert(data); - assert(arg); - - ravl_data_t *node_data = data; - block_t *block = node_data->value; - assert(block); - - debug_cb_args_t *cb_args = (debug_cb_args_t *)arg; - coarse_memory_provider_t *provider = cb_args->provider; - - ravl_node_t *node = - ravl_find(provider->all_blocks, data, RAVL_PREDICATE_EQUAL); - assert(node); - - block_t *block_next = get_block_next(node); - block_t *block_prev = get_block_prev(node); - - cb_args->num_all_blocks++; - if (!block->used) { - cb_args->num_free_blocks++; - } - - assert(block->data); - assert(block->size > 0); - - // There shouldn't be two adjacent unused blocks - // if they are continuous and have the same origin. - if (block_prev && !block_prev->used && !block->used && - (block_prev->data + block_prev->size == block->data)) { - assert(!is_same_origin(provider->upstream_blocks, block_prev, block)); - } - - if (block_next && !block_next->used && !block->used && - (block->data + block->size == block_next->data)) { - assert(!is_same_origin(provider->upstream_blocks, block, block_next)); - } - - // data addresses in the list are in ascending order - if (block_prev) { - assert(block_prev->data < block->data); - } - - if (block_next) { - assert(block->data < block_next->data); - } - - // two block's data should not overlap - if (block_next) { - assert((block->data + block->size) <= block_next->data); - } - - cb_args->sum_blocks_size += block->size; - if (block->used) { - cb_args->sum_used += block->size; - } -} - -static void debug_verify_upstream_blocks_cb(void *data, void *arg) { - assert(data); - assert(arg); - - ravl_data_t *node_data = data; - block_t *alloc = node_data->value; - assert(alloc); - - debug_cb_args_t *cb_args = (debug_cb_args_t *)arg; - coarse_memory_provider_t *provider = cb_args->provider; - - ravl_node_t *node = - ravl_find(provider->upstream_blocks, data, RAVL_PREDICATE_EQUAL); - assert(node); - - block_t *alloc_next = get_block_next(node); - block_t *alloc_prev = get_block_prev(node); - - cb_args->num_alloc_blocks++; - cb_args->sum_alloc_size += alloc->size; - - assert(alloc->data); - assert(alloc->size > 0); - - // data addresses in the list are in ascending order - if (alloc_prev) { - assert(alloc_prev->data < alloc->data); - } - - if (alloc_next) { - assert(alloc->data < alloc_next->data); - } - - // data should not overlap - if (alloc_next) { - assert((alloc->data + alloc->size) <= alloc_next->data); - } -} - -static umf_result_t -coarse_memory_provider_get_stats(void *provider, - coarse_memory_provider_stats_t *stats); - -static bool debug_check(coarse_memory_provider_t *provider) { - assert(provider); - - coarse_memory_provider_stats_t stats = {0}; - coarse_memory_provider_get_stats(provider, &stats); - - debug_cb_args_t cb_args = {0}; - cb_args.provider = provider; - - // verify the all_blocks list - ravl_foreach(provider->all_blocks, debug_verify_all_blocks_cb, &cb_args); - - assert(cb_args.num_all_blocks == stats.num_all_blocks); - assert(cb_args.num_free_blocks == stats.num_free_blocks); - assert(cb_args.sum_used == provider->used_size); - assert(cb_args.sum_blocks_size == provider->alloc_size); - assert(provider->alloc_size >= provider->used_size); - - // verify the upstream_blocks list - ravl_foreach(provider->upstream_blocks, debug_verify_upstream_blocks_cb, - &cb_args); - - assert(cb_args.sum_alloc_size == provider->alloc_size); - assert(cb_args.num_alloc_blocks == stats.num_upstream_blocks); - - return true; -} -#endif /* NDEBUG */ // end of DEBUG code - -static umf_result_t -coarse_add_upstream_block(coarse_memory_provider_t *coarse_provider, void *addr, - size_t size) { - ravl_node_t *alloc_node = NULL; - - block_t *alloc = coarse_ravl_add_new(coarse_provider->upstream_blocks, addr, - size, &alloc_node); - if (alloc == NULL) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - block_t *new_block = - coarse_ravl_add_new(coarse_provider->all_blocks, addr, size, NULL); - if (new_block == NULL) { - coarse_ravl_rm(coarse_provider->upstream_blocks, addr); - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - // check if the new upstream block can be merged with its neighbours - alloc_node = upstream_block_merge_with_prev(coarse_provider, alloc_node); - alloc_node = upstream_block_merge_with_next(coarse_provider, alloc_node); - - new_block->used = true; - coarse_provider->alloc_size += size; - coarse_provider->used_size += size; - - return UMF_RESULT_SUCCESS; -} - -static umf_result_t -coarse_memory_provider_set_name(coarse_memory_provider_t *coarse_provider) { - if (coarse_provider->upstream_memory_provider == NULL) { - // COARSE_BASE_NAME will be used - coarse_provider->name = NULL; - return UMF_RESULT_SUCCESS; - } - - const char *up_name = - umfMemoryProviderGetName(coarse_provider->upstream_memory_provider); - if (!up_name) { - return UMF_RESULT_ERROR_UNKNOWN; - } - - size_t length = - strlen(COARSE_BASE_NAME) + strlen(up_name) + 3; // + 3 for " ()" - - coarse_provider->name = umf_ba_global_alloc(length + 1); // + 1 for '\0' - if (coarse_provider->name == NULL) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - sprintf(coarse_provider->name, "%s (%s)", COARSE_BASE_NAME, up_name); - - return UMF_RESULT_SUCCESS; -} - -// needed for coarse_memory_provider_initialize() -static umf_result_t coarse_memory_provider_alloc(void *provider, size_t size, - size_t alignment, - void **resultPtr); - -// needed for coarse_memory_provider_initialize() -static umf_result_t coarse_memory_provider_free(void *provider, void *ptr, - size_t bytes); - -static umf_result_t coarse_memory_provider_initialize(void *params, - void **provider) { - assert(provider); - - if (params == NULL) { - LOG_ERR("coarse provider parameters are missing"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - coarse_memory_provider_params_t *coarse_params = - (coarse_memory_provider_params_t *)params; - - // check params - if (!coarse_params->upstream_memory_provider == - !coarse_params->init_buffer) { - LOG_ERR("either upstream provider or init buffer has to be provided in " - "the parameters (exactly one of them)"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - if (coarse_params->init_buffer_size == 0 && - (coarse_params->immediate_init_from_upstream || - coarse_params->init_buffer != NULL)) { - LOG_ERR("init_buffer_size has to be greater than 0 if " - "immediate_init_from_upstream or init_buffer is set"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - if (coarse_params->init_buffer_size != 0 && - (!coarse_params->immediate_init_from_upstream && - coarse_params->init_buffer == NULL)) { - LOG_ERR("init_buffer_size is greater than 0 but none of " - "immediate_init_from_upstream nor init_buffer is set"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - if (coarse_params->destroy_upstream_memory_provider && - !coarse_params->upstream_memory_provider) { - LOG_ERR("destroy_upstream_memory_provider is true, but an upstream " - "provider is not provided"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - coarse_memory_provider_t *coarse_provider = - umf_ba_global_alloc(sizeof(*coarse_provider)); - if (!coarse_provider) { - LOG_ERR("out of the host memory"); - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - memset(coarse_provider, 0, sizeof(*coarse_provider)); - - coarse_provider->upstream_memory_provider = - coarse_params->upstream_memory_provider; - coarse_provider->destroy_upstream_memory_provider = - coarse_params->destroy_upstream_memory_provider; - coarse_provider->allocation_strategy = coarse_params->allocation_strategy; - coarse_provider->init_buffer = coarse_params->init_buffer; - - if (coarse_provider->upstream_memory_provider) { - coarse_provider->disable_upstream_provider_free = - umfIsFreeOpDefault(coarse_provider->upstream_memory_provider); - } else { - coarse_provider->disable_upstream_provider_free = false; - } - - umf_result_t umf_result = coarse_memory_provider_set_name(coarse_provider); - if (umf_result != UMF_RESULT_SUCCESS) { - LOG_ERR("name initialization failed"); - goto err_free_coarse_provider; - } - - // most of the error handling paths below set this error - umf_result = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - - coarse_provider->upstream_blocks = - ravl_new_sized(coarse_ravl_comp, sizeof(ravl_data_t)); - if (coarse_provider->upstream_blocks == NULL) { - LOG_ERR("out of the host memory"); - goto err_free_name; - } - - coarse_provider->free_blocks = - ravl_new_sized(coarse_ravl_comp, sizeof(ravl_data_t)); - if (coarse_provider->free_blocks == NULL) { - LOG_ERR("out of the host memory"); - goto err_delete_ravl_upstream_blocks; - } - - coarse_provider->all_blocks = - ravl_new_sized(coarse_ravl_comp, sizeof(ravl_data_t)); - if (coarse_provider->all_blocks == NULL) { - LOG_ERR("out of the host memory"); - goto err_delete_ravl_free_blocks; - } - - coarse_provider->alloc_size = 0; - coarse_provider->used_size = 0; - - if (utils_mutex_init(&coarse_provider->lock) == NULL) { - LOG_ERR("lock initialization failed"); - umf_result = UMF_RESULT_ERROR_UNKNOWN; - goto err_delete_ravl_all_blocks; - } - - if (coarse_params->upstream_memory_provider && - coarse_params->immediate_init_from_upstream) { - // allocate and immediately deallocate memory using the upstream provider - void *init_buffer = NULL; - coarse_memory_provider_alloc( - coarse_provider, coarse_params->init_buffer_size, 0, &init_buffer); - if (init_buffer == NULL) { - goto err_destroy_mutex; - } - - coarse_memory_provider_free(coarse_provider, init_buffer, - coarse_params->init_buffer_size); - - } else if (coarse_params->init_buffer) { - umf_result = coarse_add_upstream_block(coarse_provider, - coarse_provider->init_buffer, - coarse_params->init_buffer_size); - if (umf_result != UMF_RESULT_SUCCESS) { - goto err_destroy_mutex; - } - - LOG_DEBUG("coarse_ALLOC (init_buffer) %zu used %zu alloc %zu", - coarse_params->init_buffer_size, coarse_provider->used_size, - coarse_provider->alloc_size); - - coarse_memory_provider_free(coarse_provider, - coarse_provider->init_buffer, - coarse_params->init_buffer_size); - } - - assert(coarse_provider->used_size == 0); - assert(coarse_provider->alloc_size == coarse_params->init_buffer_size); - assert(debug_check(coarse_provider)); - - *provider = coarse_provider; - - return UMF_RESULT_SUCCESS; - -err_destroy_mutex: - utils_mutex_destroy_not_free(&coarse_provider->lock); -err_delete_ravl_all_blocks: - ravl_delete(coarse_provider->all_blocks); -err_delete_ravl_free_blocks: - ravl_delete(coarse_provider->free_blocks); -err_delete_ravl_upstream_blocks: - ravl_delete(coarse_provider->upstream_blocks); -err_free_name: - umf_ba_global_free(coarse_provider->name); -err_free_coarse_provider: - umf_ba_global_free(coarse_provider); - return umf_result; -} - -static void coarse_ravl_cb_rm_upstream_blocks_node(void *data, void *arg) { - assert(data); - assert(arg); - - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)arg; - ravl_data_t *node_data = data; - block_t *alloc = node_data->value; - assert(alloc); - - if (coarse_provider->upstream_memory_provider && - !coarse_provider->disable_upstream_provider_free) { - // We continue to deallocate alloc blocks even if the upstream provider doesn't return success. - umfMemoryProviderFree(coarse_provider->upstream_memory_provider, - alloc->data, alloc->size); - } - - assert(coarse_provider->alloc_size >= alloc->size); - coarse_provider->alloc_size -= alloc->size; - - umf_ba_global_free(alloc); -} - -static void coarse_ravl_cb_rm_all_blocks_node(void *data, void *arg) { - assert(data); - assert(arg); - - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)arg; - ravl_data_t *node_data = data; - block_t *block = node_data->value; - assert(block); - - if (block->used) { - assert(coarse_provider->used_size >= block->size); - coarse_provider->used_size -= block->size; - } - - if (block->free_list_ptr) { - free_blocks_rm_node(coarse_provider->free_blocks, block->free_list_ptr); - } - - umf_ba_global_free(block); -} - -static void coarse_memory_provider_finalize(void *provider) { - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - utils_mutex_destroy_not_free(&coarse_provider->lock); - - ravl_foreach(coarse_provider->all_blocks, coarse_ravl_cb_rm_all_blocks_node, - coarse_provider); - assert(coarse_provider->used_size == 0); - - ravl_foreach(coarse_provider->upstream_blocks, - coarse_ravl_cb_rm_upstream_blocks_node, coarse_provider); - assert(coarse_provider->alloc_size == 0); - - ravl_delete(coarse_provider->upstream_blocks); - ravl_delete(coarse_provider->all_blocks); - ravl_delete(coarse_provider->free_blocks); - - umf_ba_global_free(coarse_provider->name); - - if (coarse_provider->destroy_upstream_memory_provider && - coarse_provider->upstream_memory_provider) { - umfMemoryProviderDestroy(coarse_provider->upstream_memory_provider); - } - - umf_ba_global_free(coarse_provider); -} - -static umf_result_t -create_aligned_block(coarse_memory_provider_t *coarse_provider, - size_t orig_size, size_t alignment, block_t **current) { - (void)orig_size; // unused in the Release version - int rv; - - block_t *curr = *current; - - // In case of non-zero alignment create an aligned block what would be further used. - uintptr_t orig_data = (uintptr_t)curr->data; - uintptr_t aligned_data = ALIGN_UP(orig_data, alignment); - size_t padding = aligned_data - orig_data; - if (alignment > 0 && padding > 0) { - block_t *aligned_block = coarse_ravl_add_new( - coarse_provider->all_blocks, curr->data + padding, - curr->size - padding, NULL); - if (aligned_block == NULL) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - curr->used = false; - curr->size = padding; - - rv = free_blocks_add(coarse_provider->free_blocks, curr); - if (rv) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - // use aligned block - *current = aligned_block; - assert((*current)->size >= orig_size); - } - - return UMF_RESULT_SUCCESS; -} - -// Split the current block and put the new block after the one that we use. -static umf_result_t -split_current_block(coarse_memory_provider_t *coarse_provider, block_t *curr, - size_t size) { - ravl_node_t *new_node = NULL; - - block_t *new_block = - coarse_ravl_add_new(coarse_provider->all_blocks, curr->data + size, - curr->size - size, &new_node); - if (new_block == NULL) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - new_block->used = false; - - int rv = - free_blocks_add(coarse_provider->free_blocks, get_node_block(new_node)); - if (rv) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - return UMF_RESULT_SUCCESS; -} - -static block_t * -find_free_block(struct ravl *free_blocks, size_t size, size_t alignment, - coarse_memory_provider_strategy_t allocation_strategy) { - block_t *block; - - switch (allocation_strategy) { - case UMF_COARSE_MEMORY_STRATEGY_FASTEST: - // Always allocate a free block of the (size + alignment) size - // and later cut out the properly aligned part leaving two remaining parts. - return free_blocks_rm_ge(free_blocks, size + alignment, 0, - CHECK_ONLY_THE_FIRST_BLOCK); - - case UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE: - // First check if the first free block of the 'size' size has the correct alignment. - block = free_blocks_rm_ge(free_blocks, size, alignment, - CHECK_ONLY_THE_FIRST_BLOCK); - if (block) { - return block; - } - - // If not, use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. - return free_blocks_rm_ge(free_blocks, size + alignment, 0, - CHECK_ONLY_THE_FIRST_BLOCK); - - case UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE: - // First look through all free blocks of the 'size' size - // and choose the first one with the correct alignment. - block = free_blocks_rm_ge(free_blocks, size, alignment, - CHECK_ALL_BLOCKS_OF_SIZE); - if (block) { - return block; - } - - // If none of them had the correct alignment, - // use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. - return free_blocks_rm_ge(free_blocks, size + alignment, 0, - CHECK_ONLY_THE_FIRST_BLOCK); - - // unknown memory allocation strategy - default: - abort(); - } -} - -static umf_result_t coarse_memory_provider_alloc(void *provider, size_t size, - size_t alignment, - void **resultPtr) { - umf_result_t umf_result = UMF_RESULT_ERROR_UNKNOWN; - - if (resultPtr == NULL) { - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - if (utils_mutex_lock(&coarse_provider->lock) != 0) { - LOG_ERR("locking the lock failed"); - return UMF_RESULT_ERROR_UNKNOWN; - } - - assert(debug_check(coarse_provider)); - - // Find a block with greater or equal size using the given memory allocation strategy - block_t *curr = - find_free_block(coarse_provider->free_blocks, size, alignment, - coarse_provider->allocation_strategy); - - // If the block that we want to reuse has a greater size, split it. - // Try to merge the split part with the successor if it is not used. - enum { ACTION_NONE = 0, ACTION_USE, ACTION_SPLIT } action = ACTION_NONE; - - if (curr && curr->size > size) { - action = ACTION_SPLIT; - } else if (curr && curr->size == size) { - action = ACTION_USE; - } - - if (action) { // ACTION_SPLIT or ACTION_USE - assert(curr->used == false); - - // In case of non-zero alignment create an aligned block what would be further used. - if (alignment > 0) { - umf_result = - create_aligned_block(coarse_provider, size, alignment, &curr); - if (umf_result != UMF_RESULT_SUCCESS) { - utils_mutex_unlock(&coarse_provider->lock); - return umf_result; - } - } - - if (action == ACTION_SPLIT) { - // Split the current block and put the new block after the one that we use. - umf_result = split_current_block(coarse_provider, curr, size); - if (umf_result != UMF_RESULT_SUCCESS) { - utils_mutex_unlock(&coarse_provider->lock); - return umf_result; - } - - curr->size = size; - - LOG_DEBUG("coarse_ALLOC (split_block) %zu used %zu alloc %zu", size, - coarse_provider->used_size, coarse_provider->alloc_size); - - } else { // action == ACTION_USE - LOG_DEBUG("coarse_ALLOC (same_block) %zu used %zu alloc %zu", size, - coarse_provider->used_size, coarse_provider->alloc_size); - } - - curr->used = true; - *resultPtr = curr->data; - coarse_provider->used_size += size; - - assert(debug_check(coarse_provider)); - utils_mutex_unlock(&coarse_provider->lock); - - return UMF_RESULT_SUCCESS; - } - - // no suitable block found - try to get more memory from the upstream provider - umf_result = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - - if (coarse_provider->upstream_memory_provider == NULL) { - LOG_ERR("out of memory - no upstream memory provider given"); - goto err_unlock; - } - - umfMemoryProviderAlloc(coarse_provider->upstream_memory_provider, size, - alignment, resultPtr); - if (*resultPtr == NULL) { - LOG_ERR("out of memory - upstream memory provider allocation failed"); - goto err_unlock; - } - - ASSERT_IS_ALIGNED(((uintptr_t)(*resultPtr)), alignment); - - umf_result = coarse_add_upstream_block(coarse_provider, *resultPtr, size); - if (umf_result != UMF_RESULT_SUCCESS) { - if (!coarse_provider->disable_upstream_provider_free) { - umfMemoryProviderFree(coarse_provider->upstream_memory_provider, - *resultPtr, size); - } - goto err_unlock; - } - - LOG_DEBUG("coarse_ALLOC (upstream) %zu used %zu alloc %zu", size, - coarse_provider->used_size, coarse_provider->alloc_size); - - umf_result = UMF_RESULT_SUCCESS; - -err_unlock: - assert(debug_check(coarse_provider)); - utils_mutex_unlock(&coarse_provider->lock); - - return umf_result; -} - -static umf_result_t coarse_memory_provider_free(void *provider, void *ptr, - size_t bytes) { - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - if (utils_mutex_lock(&coarse_provider->lock) != 0) { - LOG_ERR("locking the lock failed"); - return UMF_RESULT_ERROR_UNKNOWN; - } - - assert(debug_check(coarse_provider)); - - ravl_node_t *node = coarse_ravl_find_node(coarse_provider->all_blocks, ptr); - if (node == NULL) { - // the block was not found - utils_mutex_unlock(&coarse_provider->lock); - LOG_ERR("memory block not found (ptr = %p, size = %zu)", ptr, bytes); - return UMF_RESULT_ERROR_UNKNOWN; - } - - block_t *block = get_node_block(node); - if (!block->used) { - // the block is already free - utils_mutex_unlock(&coarse_provider->lock); - LOG_ERR("the block is already free"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - if (bytes > 0 && bytes != block->size) { - // wrong size of allocation - utils_mutex_unlock(&coarse_provider->lock); - LOG_ERR("wrong size of allocation"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - LOG_DEBUG("coarse_FREE (return_block_to_pool) %zu used %zu alloc %zu", - block->size, coarse_provider->used_size - block->size, - coarse_provider->alloc_size); - - assert(coarse_provider->used_size >= block->size); - coarse_provider->used_size -= block->size; - - block->used = false; - - // Merge with prev and/or next block if they are unused and have continuous data. - node = free_block_merge_with_prev(coarse_provider, node); - node = free_block_merge_with_next(coarse_provider, node); - - int rv = - free_blocks_add(coarse_provider->free_blocks, get_node_block(node)); - if (rv) { - utils_mutex_unlock(&coarse_provider->lock); - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - assert(debug_check(coarse_provider)); - utils_mutex_unlock(&coarse_provider->lock); - - return UMF_RESULT_SUCCESS; -} - -static void coarse_memory_provider_get_last_native_error(void *provider, - const char **ppMessage, - int32_t *pError) { - (void)provider; // unused - - if (ppMessage == NULL || pError == NULL) { - assert(0); - return; - } - - // Nothing more is needed here, since - // there is no UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC error used. -} - -static umf_result_t coarse_memory_provider_get_min_page_size(void *provider, - void *ptr, - size_t *pageSize) { - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - if (!coarse_provider->upstream_memory_provider) { - *pageSize = utils_get_page_size(); - return UMF_RESULT_SUCCESS; - } - - return umfMemoryProviderGetMinPageSize( - coarse_provider->upstream_memory_provider, ptr, pageSize); -} - -static umf_result_t -coarse_memory_provider_get_recommended_page_size(void *provider, size_t size, - size_t *pageSize) { - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - if (!coarse_provider->upstream_memory_provider) { - *pageSize = utils_get_page_size(); - return UMF_RESULT_SUCCESS; - } - - return umfMemoryProviderGetRecommendedPageSize( - coarse_provider->upstream_memory_provider, size, pageSize); -} - -static const char *coarse_memory_provider_get_name(void *provider) { - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - if (!coarse_provider->name) { - return COARSE_BASE_NAME; - } - - return coarse_provider->name; -} - -static void ravl_cb_count(void *data, void *arg) { - assert(arg); - (void)data; /* unused */ - - size_t *num_all_blocks = arg; - (*num_all_blocks)++; -} - -static void ravl_cb_count_free(void *data, void *arg) { - assert(data); - assert(arg); - - ravl_data_t *node_data = data; - assert(node_data); - ravl_free_blocks_head_t *head_node = node_data->value; - assert(head_node); - struct ravl_free_blocks_elem_t *free_block = head_node->head; - assert(free_block); - - size_t *num_all_blocks = arg; - while (free_block) { - (*num_all_blocks)++; - free_block = free_block->next; - } -} - -static umf_result_t -coarse_memory_provider_get_stats(void *provider, - coarse_memory_provider_stats_t *stats) { - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - // count blocks - size_t num_upstream_blocks = 0; - ravl_foreach(coarse_provider->upstream_blocks, ravl_cb_count, - &num_upstream_blocks); - - size_t num_all_blocks = 0; - ravl_foreach(coarse_provider->all_blocks, ravl_cb_count, &num_all_blocks); - - size_t num_free_blocks = 0; - ravl_foreach(coarse_provider->free_blocks, ravl_cb_count_free, - &num_free_blocks); - - stats->alloc_size = coarse_provider->alloc_size; - stats->used_size = coarse_provider->used_size; - stats->num_upstream_blocks = num_upstream_blocks; - stats->num_all_blocks = num_all_blocks; - stats->num_free_blocks = num_free_blocks; - - return UMF_RESULT_SUCCESS; -} - -static umf_result_t coarse_memory_provider_purge_lazy(void *provider, void *ptr, - size_t size) { - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - if (coarse_provider->upstream_memory_provider == NULL) { - LOG_ERR("no upstream memory provider given"); - return UMF_RESULT_ERROR_NOT_SUPPORTED; - } - - return umfMemoryProviderPurgeLazy(coarse_provider->upstream_memory_provider, - ptr, size); -} - -static umf_result_t coarse_memory_provider_purge_force(void *provider, - void *ptr, size_t size) { - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - if (coarse_provider->upstream_memory_provider == NULL) { - LOG_ERR("no upstream memory provider given"); - return UMF_RESULT_ERROR_NOT_SUPPORTED; - } - - return umfMemoryProviderPurgeForce( - coarse_provider->upstream_memory_provider, ptr, size); -} - -static umf_result_t coarse_memory_provider_allocation_split(void *provider, - void *ptr, - size_t totalSize, - size_t firstSize) { - umf_result_t umf_result; - - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - if (utils_mutex_lock(&coarse_provider->lock) != 0) { - LOG_ERR("locking the lock failed"); - return UMF_RESULT_ERROR_UNKNOWN; - } - - assert(debug_check(coarse_provider)); - - ravl_node_t *node = coarse_ravl_find_node(coarse_provider->all_blocks, ptr); - if (node == NULL) { - LOG_ERR("memory block not found"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - block_t *block = get_node_block(node); - - if (block->size != totalSize) { - LOG_ERR("wrong totalSize"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - if (!block->used) { - LOG_ERR("block is not allocated"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - block_t *new_block = coarse_ravl_add_new(coarse_provider->all_blocks, - block->data + firstSize, - block->size - firstSize, NULL); - if (new_block == NULL) { - umf_result = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - goto err_mutex_unlock; - } - - block->size = firstSize; - new_block->used = true; - - assert(new_block->size == (totalSize - firstSize)); - - umf_result = UMF_RESULT_SUCCESS; - -err_mutex_unlock: - assert(debug_check(coarse_provider)); - utils_mutex_unlock(&coarse_provider->lock); - - return umf_result; -} - -static umf_result_t coarse_memory_provider_allocation_merge(void *provider, - void *lowPtr, - void *highPtr, - size_t totalSize) { - umf_result_t umf_result; - - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - if (utils_mutex_lock(&coarse_provider->lock) != 0) { - LOG_ERR("locking the lock failed"); - return UMF_RESULT_ERROR_UNKNOWN; - } - - assert(debug_check(coarse_provider)); - - ravl_node_t *low_node = - coarse_ravl_find_node(coarse_provider->all_blocks, lowPtr); - if (low_node == NULL) { - LOG_ERR("the lowPtr memory block not found"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - block_t *low_block = get_node_block(low_node); - if (!low_block->used) { - LOG_ERR("the lowPtr block is not allocated"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - ravl_node_t *high_node = - coarse_ravl_find_node(coarse_provider->all_blocks, highPtr); - if (high_node == NULL) { - LOG_ERR("the highPtr memory block not found"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - block_t *high_block = get_node_block(high_node); - if (!high_block->used) { - LOG_ERR("the highPtr block is not allocated"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - if (get_node_next(low_node) != high_node) { - LOG_ERR("given pointers cannot be merged"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - if (get_node_prev(high_node) != low_node) { - LOG_ERR("given pointers cannot be merged"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - if (low_block->size + high_block->size != totalSize) { - LOG_ERR("wrong totalSize"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - if ((uintptr_t)highPtr != ((uintptr_t)lowPtr + low_block->size)) { - LOG_ERR("given pointers cannot be merged"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - ravl_node_t *merged_node = NULL; - - umf_result = user_block_merge(coarse_provider, low_node, high_node, true, - &merged_node); - if (umf_result != UMF_RESULT_SUCCESS) { - LOG_ERR("merging failed"); - goto err_mutex_unlock; - } - - assert(merged_node == low_node); - assert(low_block->size == totalSize); - - umf_result = UMF_RESULT_SUCCESS; - -err_mutex_unlock: - assert(debug_check(coarse_provider)); - utils_mutex_unlock(&coarse_provider->lock); - - return umf_result; -} - -umf_memory_provider_ops_t UMF_COARSE_MEMORY_PROVIDER_OPS = { - .version = UMF_VERSION_CURRENT, - .initialize = coarse_memory_provider_initialize, - .finalize = coarse_memory_provider_finalize, - .alloc = coarse_memory_provider_alloc, - .get_last_native_error = coarse_memory_provider_get_last_native_error, - .get_recommended_page_size = - coarse_memory_provider_get_recommended_page_size, - .get_min_page_size = coarse_memory_provider_get_min_page_size, - .get_name = coarse_memory_provider_get_name, - .ext.free = coarse_memory_provider_free, - .ext.purge_lazy = coarse_memory_provider_purge_lazy, - .ext.purge_force = coarse_memory_provider_purge_force, - .ext.allocation_merge = coarse_memory_provider_allocation_merge, - .ext.allocation_split = coarse_memory_provider_allocation_split, - // TODO - /* - .ipc.get_ipc_handle_size = coarse_memory_provider_get_ipc_handle_size, - .ipc.get_ipc_handle = coarse_memory_provider_get_ipc_handle, - .ipc.put_ipc_handle = coarse_memory_provider_put_ipc_handle, - .ipc.open_ipc_handle = coarse_memory_provider_open_ipc_handle, - .ipc.close_ipc_handle = coarse_memory_provider_close_ipc_handle, - */ -}; - -umf_memory_provider_ops_t *umfCoarseMemoryProviderOps(void) { - return &UMF_COARSE_MEMORY_PROVIDER_OPS; -} - -coarse_memory_provider_stats_t -umfCoarseMemoryProviderGetStats(umf_memory_provider_handle_t provider) { - coarse_memory_provider_stats_t stats = {0}; - - if (provider == NULL) { - return stats; - } - - void *priv = umfMemoryProviderGetPriv(provider); - - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)priv; - - if (utils_mutex_lock(&coarse_provider->lock) != 0) { - LOG_ERR("locking the lock failed"); - return stats; - } - - coarse_memory_provider_get_stats(priv, &stats); - - utils_mutex_unlock(&coarse_provider->lock); - - return stats; -} diff --git a/src/provider/provider_cuda.c b/src/provider/provider_cuda.c index ce2f1debb7..350bd016fd 100644 --- a/src/provider/provider_cuda.c +++ b/src/provider/provider_cuda.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -313,7 +313,7 @@ static umf_result_t cu_memory_provider_initialize(void *params, CUmemAllocationProp allocProps = {0}; allocProps.location.type = CU_MEM_LOCATION_TYPE_DEVICE; allocProps.type = CU_MEM_ALLOCATION_TYPE_PINNED; - allocProps.location.id = cu_provider->device; + allocProps.location.id = cu_params->cuda_device_handle; CUresult cu_result = g_cu_ops.cuMemGetAllocationGranularity( &min_alignment, &allocProps, CU_MEM_ALLOC_GRANULARITY_MINIMUM); if (cu_result != CUDA_SUCCESS) { @@ -607,11 +607,11 @@ static struct umf_memory_provider_ops_t UMF_CUDA_MEMORY_PROVIDER_OPS = { .initialize = cu_memory_provider_initialize, .finalize = cu_memory_provider_finalize, .alloc = cu_memory_provider_alloc, + .free = cu_memory_provider_free, .get_last_native_error = cu_memory_provider_get_last_native_error, .get_recommended_page_size = cu_memory_provider_get_recommended_page_size, .get_min_page_size = cu_memory_provider_get_min_page_size, .get_name = cu_memory_provider_get_name, - .ext.free = cu_memory_provider_free, // TODO /* .ext.purge_lazy = cu_memory_provider_purge_lazy, diff --git a/src/provider/provider_devdax_memory.c b/src/provider/provider_devdax_memory.c index cb5a4af572..4841a99199 100644 --- a/src/provider/provider_devdax_memory.c +++ b/src/provider/provider_devdax_memory.c @@ -65,6 +65,7 @@ umf_result_t umfDevDaxMemoryProviderParamsSetProtection( #else // !defined(_WIN32) && !defined(UMF_NO_HWLOC) #include "base_alloc_global.h" +#include "coarse.h" #include "libumf.h" #include "utils_common.h" #include "utils_concurrency.h" @@ -81,6 +82,7 @@ typedef struct devdax_memory_provider_t { size_t offset; // offset in the file used for memory mapping utils_mutex_t lock; // lock of ptr and offset unsigned protection; // combination of OS-specific protection flags + coarse_t *coarse; // coarse library handle } devdax_memory_provider_t; // DevDax Memory provider settings struct @@ -140,6 +142,12 @@ devdax_translate_params(umf_devdax_memory_provider_params_t *in_params, return UMF_RESULT_SUCCESS; } +static umf_result_t devdax_allocation_split_cb(void *provider, void *ptr, + size_t totalSize, + size_t firstSize); +static umf_result_t devdax_allocation_merge_cb(void *provider, void *lowPtr, + void *highPtr, size_t totalSize); + static umf_result_t devdax_initialize(void *params, void **provider) { umf_result_t ret; @@ -168,21 +176,42 @@ static umf_result_t devdax_initialize(void *params, void **provider) { memset(devdax_provider, 0, sizeof(*devdax_provider)); - ret = devdax_translate_params(in_params, devdax_provider); + coarse_params_t coarse_params = {0}; + coarse_params.provider = devdax_provider; + coarse_params.page_size = DEVDAX_PAGE_SIZE_2MB; + // The alloc callback is not available in case of the devdax provider + // because it is a fixed-size memory provider + // and the entire devdax memory is added as a single block + // to the coarse library. + coarse_params.cb.alloc = NULL; + coarse_params.cb.free = NULL; // not available for the devdax provider + coarse_params.cb.split = devdax_allocation_split_cb; + coarse_params.cb.merge = devdax_allocation_merge_cb; + + coarse_t *coarse = NULL; + ret = coarse_new(&coarse_params, &coarse); if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR("coarse_new() failed"); goto err_free_devdax_provider; } + devdax_provider->coarse = coarse; + + ret = devdax_translate_params(in_params, devdax_provider); + if (ret != UMF_RESULT_SUCCESS) { + goto err_coarse_delete; + } + devdax_provider->size = in_params->size; if (utils_copy_path(in_params->path, devdax_provider->path, PATH_MAX)) { - goto err_free_devdax_provider; + goto err_coarse_delete; } int fd = utils_devdax_open(in_params->path); if (fd == -1) { LOG_ERR("cannot open the device DAX: %s", in_params->path); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_free_devdax_provider; + goto err_coarse_delete; } bool is_dax = false; @@ -196,23 +225,26 @@ static umf_result_t devdax_initialize(void *params, void **provider) { LOG_PDEBUG("mapping the devdax failed (path=%s, size=%zu)", in_params->path, devdax_provider->size); ret = UMF_RESULT_ERROR_UNKNOWN; - goto err_free_devdax_provider; + goto err_coarse_delete; } if (!is_dax) { LOG_ERR("mapping the devdax with MAP_SYNC failed: %s", in_params->path); ret = UMF_RESULT_ERROR_UNKNOWN; - - if (devdax_provider->base) { - utils_munmap(devdax_provider->base, devdax_provider->size); - } - - goto err_free_devdax_provider; + goto err_unmap_devdax; } LOG_DEBUG("devdax memory mapped (path=%s, size=%zu, addr=%p)", in_params->path, devdax_provider->size, devdax_provider->base); + // add the entire devdax memory as a single block + ret = coarse_add_memory_fixed(coarse, devdax_provider->base, + devdax_provider->size); + if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR("adding memory block failed"); + goto err_unmap_devdax; + } + if (utils_mutex_init(&devdax_provider->lock) == NULL) { LOG_ERR("lock init failed"); ret = UMF_RESULT_ERROR_UNKNOWN; @@ -224,7 +256,11 @@ static umf_result_t devdax_initialize(void *params, void **provider) { return UMF_RESULT_SUCCESS; err_unmap_devdax: - utils_munmap(devdax_provider->base, devdax_provider->size); + if (devdax_provider->base) { + utils_munmap(devdax_provider->base, devdax_provider->size); + } +err_coarse_delete: + coarse_delete(devdax_provider->coarse); err_free_devdax_provider: umf_ba_global_free(devdax_provider); return ret; @@ -234,78 +270,15 @@ static void devdax_finalize(void *provider) { devdax_memory_provider_t *devdax_provider = provider; utils_mutex_destroy_not_free(&devdax_provider->lock); utils_munmap(devdax_provider->base, devdax_provider->size); + coarse_delete(devdax_provider->coarse); umf_ba_global_free(devdax_provider); } -static int devdax_alloc_aligned(size_t length, size_t alignment, void *base, - size_t size, utils_mutex_t *lock, - void **out_addr, size_t *offset) { - assert(out_addr); - - if (utils_mutex_lock(lock)) { - LOG_ERR("locking file offset failed"); - return -1; - } - - uintptr_t ptr = (uintptr_t)base + *offset; - uintptr_t rest_of_div = alignment ? (ptr % alignment) : 0; - - if (alignment > 0 && rest_of_div > 0) { - ptr += alignment - rest_of_div; - } - - size_t new_offset = ptr - (uintptr_t)base + length; - - if (new_offset > size) { - utils_mutex_unlock(lock); - LOG_ERR("cannot allocate more memory than the device DAX size: %zu", - size); - return -1; - } - - *offset = new_offset; - *out_addr = (void *)ptr; - - utils_mutex_unlock(lock); - - return 0; -} - static umf_result_t devdax_alloc(void *provider, size_t size, size_t alignment, void **resultPtr) { - int ret; - - // alignment must be a power of two and a multiple or a divider of the page size - if (alignment && ((alignment & (alignment - 1)) || - ((alignment % DEVDAX_PAGE_SIZE_2MB) && - (DEVDAX_PAGE_SIZE_2MB % alignment)))) { - LOG_ERR("wrong alignment: %zu (not a power of 2 or a multiple or a " - "divider of the page size (%zu))", - alignment, DEVDAX_PAGE_SIZE_2MB); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - if (IS_NOT_ALIGNED(alignment, DEVDAX_PAGE_SIZE_2MB)) { - alignment = ALIGN_UP(alignment, DEVDAX_PAGE_SIZE_2MB); - } - devdax_memory_provider_t *devdax_provider = (devdax_memory_provider_t *)provider; - - void *addr = NULL; - errno = 0; - ret = devdax_alloc_aligned(size, alignment, devdax_provider->base, - devdax_provider->size, &devdax_provider->lock, - &addr, &devdax_provider->offset); - if (ret) { - devdax_store_last_native_error(UMF_DEVDAX_RESULT_ERROR_ALLOC_FAILED, 0); - LOG_ERR("memory allocation failed"); - return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; - } - - *resultPtr = addr; - - return UMF_RESULT_SUCCESS; + return coarse_alloc(devdax_provider->coarse, size, alignment, resultPtr); } static void devdax_get_last_native_error(void *provider, const char **ppMessage, @@ -391,6 +364,14 @@ static const char *devdax_get_name(void *provider) { static umf_result_t devdax_allocation_split(void *provider, void *ptr, size_t totalSize, size_t firstSize) { + devdax_memory_provider_t *devdax_provider = + (devdax_memory_provider_t *)provider; + return coarse_split(devdax_provider->coarse, ptr, totalSize, firstSize); +} + +static umf_result_t devdax_allocation_split_cb(void *provider, void *ptr, + size_t totalSize, + size_t firstSize) { (void)provider; (void)ptr; (void)totalSize; @@ -400,6 +381,14 @@ static umf_result_t devdax_allocation_split(void *provider, void *ptr, static umf_result_t devdax_allocation_merge(void *provider, void *lowPtr, void *highPtr, size_t totalSize) { + devdax_memory_provider_t *devdax_provider = + (devdax_memory_provider_t *)provider; + return coarse_merge(devdax_provider->coarse, lowPtr, highPtr, totalSize); +} + +static umf_result_t devdax_allocation_merge_cb(void *provider, void *lowPtr, + void *highPtr, + size_t totalSize) { (void)provider; (void)lowPtr; (void)highPtr; @@ -534,11 +523,18 @@ static umf_result_t devdax_close_ipc_handle(void *provider, void *ptr, return UMF_RESULT_SUCCESS; } +static umf_result_t devdax_free(void *provider, void *ptr, size_t size) { + devdax_memory_provider_t *devdax_provider = + (devdax_memory_provider_t *)provider; + return coarse_free(devdax_provider->coarse, ptr, size); +} + static umf_memory_provider_ops_t UMF_DEVDAX_MEMORY_PROVIDER_OPS = { .version = UMF_VERSION_CURRENT, .initialize = devdax_initialize, .finalize = devdax_finalize, .alloc = devdax_alloc, + .free = devdax_free, .get_last_native_error = devdax_get_last_native_error, .get_recommended_page_size = devdax_get_recommended_page_size, .get_min_page_size = devdax_get_min_page_size, diff --git a/src/provider/provider_file_memory.c b/src/provider/provider_file_memory.c index 7c9ee38564..ea69dc7b62 100644 --- a/src/provider/provider_file_memory.c +++ b/src/provider/provider_file_memory.c @@ -71,6 +71,7 @@ umf_result_t umfFileMemoryProviderParamsSetVisibility( #else // !defined(_WIN32) && !defined(UMF_NO_HWLOC) #include "base_alloc_global.h" +#include "coarse.h" #include "critnib.h" #include "libumf.h" #include "utils_common.h" @@ -109,6 +110,8 @@ typedef struct file_memory_provider_t { // It is needed mainly in the get_ipc_handle and open_ipc_handle hooks // to mmap a specific part of a file. critnib *fd_offset_map; + + coarse_t *coarse; // coarse library handle } file_memory_provider_t; // File Memory Provider settings struct @@ -174,6 +177,14 @@ file_translate_params(umf_file_memory_provider_params_t *in_params, return UMF_RESULT_SUCCESS; } +static umf_result_t file_alloc_cb(void *provider, size_t size, size_t alignment, + void **resultPtr); +static umf_result_t file_allocation_split_cb(void *provider, void *ptr, + size_t totalSize, + size_t firstSize); +static umf_result_t file_allocation_merge_cb(void *provider, void *lowPtr, + void *highPtr, size_t totalSize); + static umf_result_t file_initialize(void *params, void **provider) { umf_result_t ret; @@ -241,10 +252,27 @@ static umf_result_t file_initialize(void *params, void **provider) { file_provider->page_size = utils_get_page_size(); } + coarse_params_t coarse_params = {0}; + coarse_params.provider = file_provider; + coarse_params.page_size = file_provider->page_size; + coarse_params.cb.alloc = file_alloc_cb; + coarse_params.cb.free = NULL; // not available for the file provider + coarse_params.cb.split = file_allocation_split_cb; + coarse_params.cb.merge = file_allocation_merge_cb; + + coarse_t *coarse = NULL; + ret = coarse_new(&coarse_params, &coarse); + if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR("coarse_new() failed"); + goto err_close_fd; + } + + file_provider->coarse = coarse; + if (utils_mutex_init(&file_provider->lock) == NULL) { LOG_ERR("lock init failed"); ret = UMF_RESULT_ERROR_UNKNOWN; - goto err_close_fd; + goto err_coarse_delete; } file_provider->fd_offset_map = critnib_new(); @@ -269,6 +297,8 @@ static umf_result_t file_initialize(void *params, void **provider) { critnib_delete(file_provider->fd_offset_map); err_mutex_destroy_not_free: utils_mutex_destroy_not_free(&file_provider->lock); +err_coarse_delete: + coarse_delete(file_provider->coarse); err_close_fd: utils_close_fd(file_provider->fd); err_free_file_provider: @@ -293,6 +323,7 @@ static void file_finalize(void *provider) { utils_close_fd(file_provider->fd); critnib_delete(file_provider->fd_offset_map); critnib_delete(file_provider->mmaps); + coarse_delete(file_provider->coarse); umf_ba_global_free(file_provider); } @@ -451,11 +482,19 @@ static umf_result_t file_alloc_aligned(file_memory_provider_t *file_provider, static umf_result_t file_alloc(void *provider, size_t size, size_t alignment, void **resultPtr) { + file_memory_provider_t *file_provider = (file_memory_provider_t *)provider; + return coarse_alloc(file_provider->coarse, size, alignment, resultPtr); +} + +static umf_result_t file_alloc_cb(void *provider, size_t size, size_t alignment, + void **resultPtr) { umf_result_t umf_result; int ret; file_memory_provider_t *file_provider = (file_memory_provider_t *)provider; + *resultPtr = NULL; + // alignment must be a power of two and a multiple or a divider of the page size if (alignment && ((alignment & (alignment - 1)) || ((alignment % file_provider->page_size) && @@ -488,8 +527,15 @@ static umf_result_t file_alloc(void *provider, size_t size, size_t alignment, LOG_ERR("inserting a value to the file descriptor offset map failed " "(addr=%p, offset=%zu)", addr, alloc_offset_fd); + // We cannot undo the file_alloc_aligned() call here, + // because the file memory provider does not support the free operation. + return UMF_RESULT_ERROR_UNKNOWN; } + LOG_DEBUG("inserted a value to the file descriptor offset map (addr=%p, " + "offset=%zu)", + addr, alloc_offset_fd); + *resultPtr = addr; return UMF_RESULT_SUCCESS; @@ -576,10 +622,15 @@ static const char *file_get_name(void *provider) { return "FILE"; } -// This function is supposed to be thread-safe, so it should NOT be called concurrently -// with file_allocation_merge() with the same pointer. static umf_result_t file_allocation_split(void *provider, void *ptr, size_t totalSize, size_t firstSize) { + file_memory_provider_t *file_provider = (file_memory_provider_t *)provider; + return coarse_split(file_provider->coarse, ptr, totalSize, firstSize); +} + +static umf_result_t file_allocation_split_cb(void *provider, void *ptr, + size_t totalSize, + size_t firstSize) { (void)totalSize; file_memory_provider_t *file_provider = (file_memory_provider_t *)provider; @@ -589,29 +640,42 @@ static umf_result_t file_allocation_split(void *provider, void *ptr, void *value = critnib_get(file_provider->fd_offset_map, (uintptr_t)ptr); if (value == NULL) { - LOG_ERR("file_allocation_split(): getting a value from the file " - "descriptor offset map failed (addr=%p)", + LOG_ERR("getting a value from the file descriptor offset map failed " + "(addr=%p)", ptr); return UMF_RESULT_ERROR_UNKNOWN; } + LOG_DEBUG("split the value from the file descriptor offset map (addr=%p) " + "from size %zu to %zu + %zu", + ptr, totalSize, firstSize, totalSize - firstSize); + uintptr_t new_key = (uintptr_t)ptr + firstSize; void *new_value = (void *)((uintptr_t)value + firstSize); int ret = critnib_insert(file_provider->fd_offset_map, new_key, new_value, 0 /* update */); if (ret) { - LOG_ERR("file_allocation_split(): inserting a value to the file " - "descriptor offset map failed (addr=%p, offset=%zu)", + LOG_ERR("inserting a value to the file descriptor offset map failed " + "(addr=%p, offset=%zu)", (void *)new_key, (size_t)new_value - 1); return UMF_RESULT_ERROR_UNKNOWN; } + LOG_DEBUG("inserted a value to the file descriptor offset map (addr=%p, " + "offset=%zu)", + (void *)new_key, (size_t)new_value - 1); + return UMF_RESULT_SUCCESS; } -// It should NOT be called concurrently with file_allocation_split() with the same pointer. static umf_result_t file_allocation_merge(void *provider, void *lowPtr, void *highPtr, size_t totalSize) { + file_memory_provider_t *file_provider = (file_memory_provider_t *)provider; + return coarse_merge(file_provider->coarse, lowPtr, highPtr, totalSize); +} + +static umf_result_t file_allocation_merge_cb(void *provider, void *lowPtr, + void *highPtr, size_t totalSize) { (void)lowPtr; (void)totalSize; @@ -623,12 +687,16 @@ static umf_result_t file_allocation_merge(void *provider, void *lowPtr, void *value = critnib_remove(file_provider->fd_offset_map, (uintptr_t)highPtr); if (value == NULL) { - LOG_ERR("file_allocation_merge(): removing a value from the file " - "descriptor offset map failed (addr=%p)", + LOG_ERR("removing a value from the file descriptor offset map failed " + "(addr=%p)", highPtr); return UMF_RESULT_ERROR_UNKNOWN; } + LOG_DEBUG("removed a value from the file descriptor offset map (addr=%p) - " + "merged with %p", + highPtr, lowPtr); + return UMF_RESULT_SUCCESS; } @@ -662,9 +730,7 @@ static umf_result_t file_get_ipc_handle(void *provider, const void *ptr, void *value = critnib_get(file_provider->fd_offset_map, (uintptr_t)ptr); if (value == NULL) { - LOG_ERR("file_get_ipc_handle(): getting a value from the IPC cache " - "failed (addr=%p)", - ptr); + LOG_ERR("getting a value from the IPC cache failed (addr=%p)", ptr); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } @@ -776,11 +842,17 @@ static umf_result_t file_close_ipc_handle(void *provider, void *ptr, return UMF_RESULT_SUCCESS; } +static umf_result_t file_free(void *provider, void *ptr, size_t size) { + file_memory_provider_t *file_provider = (file_memory_provider_t *)provider; + return coarse_free(file_provider->coarse, ptr, size); +} + static umf_memory_provider_ops_t UMF_FILE_MEMORY_PROVIDER_OPS = { .version = UMF_VERSION_CURRENT, .initialize = file_initialize, .finalize = file_finalize, .alloc = file_alloc, + .free = file_free, .get_last_native_error = file_get_last_native_error, .get_recommended_page_size = file_get_recommended_page_size, .get_min_page_size = file_get_min_page_size, diff --git a/src/provider/provider_fixed_memory.c b/src/provider/provider_fixed_memory.c new file mode 100644 index 0000000000..6392b39d39 --- /dev/null +++ b/src/provider/provider_fixed_memory.c @@ -0,0 +1,337 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "base_alloc_global.h" +#include "coarse.h" +#include "libumf.h" +#include "utils_common.h" +#include "utils_concurrency.h" +#include "utils_log.h" + +#define TLS_MSG_BUF_LEN 1024 + +typedef struct fixed_memory_provider_t { + void *base; // base address of memory + size_t size; // size of the memory region + coarse_t *coarse; // coarse library handle +} fixed_memory_provider_t; + +// Fixed Memory provider settings struct +typedef struct umf_fixed_memory_provider_params_t { + void *ptr; + size_t size; +} umf_fixed_memory_provider_params_t; + +typedef struct fixed_last_native_error_t { + int32_t native_error; + int errno_value; + char msg_buff[TLS_MSG_BUF_LEN]; +} fixed_last_native_error_t; + +static __TLS fixed_last_native_error_t TLS_last_native_error; + +// helper values used only in the Native_error_str array +#define _UMF_FIXED_RESULT_SUCCESS \ + (UMF_FIXED_RESULT_SUCCESS - UMF_FIXED_RESULT_SUCCESS) +#define _UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED \ + (UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED - UMF_FIXED_RESULT_SUCCESS) + +static const char *Native_error_str[] = { + [_UMF_FIXED_RESULT_SUCCESS] = "success", + [_UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED] = "force purging failed"}; + +static void fixed_store_last_native_error(int32_t native_error, + int errno_value) { + TLS_last_native_error.native_error = native_error; + TLS_last_native_error.errno_value = errno_value; +} + +static umf_result_t fixed_allocation_split_cb(void *provider, void *ptr, + size_t totalSize, + size_t firstSize) { + (void)provider; + (void)ptr; + (void)totalSize; + (void)firstSize; + return UMF_RESULT_SUCCESS; +} + +static umf_result_t fixed_allocation_merge_cb(void *provider, void *lowPtr, + void *highPtr, size_t totalSize) { + (void)provider; + (void)lowPtr; + (void)highPtr; + (void)totalSize; + return UMF_RESULT_SUCCESS; +} + +static umf_result_t fixed_initialize(void *params, void **provider) { + umf_result_t ret; + + if (params == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + umf_fixed_memory_provider_params_t *in_params = + (umf_fixed_memory_provider_params_t *)params; + + fixed_memory_provider_t *fixed_provider = + umf_ba_global_alloc(sizeof(*fixed_provider)); + if (!fixed_provider) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + memset(fixed_provider, 0, sizeof(*fixed_provider)); + + coarse_params_t coarse_params = {0}; + coarse_params.provider = fixed_provider; + coarse_params.page_size = utils_get_page_size(); + // The alloc callback is not available in case of the fixed provider + // because it is a fixed-size memory provider + // and the entire memory is added as a single block + // to the coarse library. + coarse_params.cb.alloc = NULL; + coarse_params.cb.free = NULL; // not available for the fixed provider + coarse_params.cb.split = fixed_allocation_split_cb; + coarse_params.cb.merge = fixed_allocation_merge_cb; + + coarse_t *coarse = NULL; + ret = coarse_new(&coarse_params, &coarse); + if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR("coarse_new() failed"); + goto err_free_fixed_provider; + } + + fixed_provider->coarse = coarse; + + fixed_provider->base = in_params->ptr; + fixed_provider->size = in_params->size; + + // add the entire memory as a single block + ret = coarse_add_memory_fixed(coarse, fixed_provider->base, + fixed_provider->size); + if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR("adding memory block failed"); + goto err_coarse_delete; + } + + *provider = fixed_provider; + + return UMF_RESULT_SUCCESS; + +err_coarse_delete: + coarse_delete(fixed_provider->coarse); +err_free_fixed_provider: + umf_ba_global_free(fixed_provider); + return ret; +} + +static void fixed_finalize(void *provider) { + fixed_memory_provider_t *fixed_provider = provider; + coarse_delete(fixed_provider->coarse); + umf_ba_global_free(fixed_provider); +} + +static umf_result_t fixed_alloc(void *provider, size_t size, size_t alignment, + void **resultPtr) { + fixed_memory_provider_t *fixed_provider = + (fixed_memory_provider_t *)provider; + + return coarse_alloc(fixed_provider->coarse, size, alignment, resultPtr); +} + +static void fixed_get_last_native_error(void *provider, const char **ppMessage, + int32_t *pError) { + (void)provider; // unused + + if (ppMessage == NULL || pError == NULL) { + assert(0); + return; + } + + *pError = TLS_last_native_error.native_error; + if (TLS_last_native_error.errno_value == 0) { + *ppMessage = Native_error_str[*pError - UMF_FIXED_RESULT_SUCCESS]; + return; + } + + const char *msg; + size_t len; + size_t pos = 0; + + msg = Native_error_str[*pError - UMF_FIXED_RESULT_SUCCESS]; + len = strlen(msg); + memcpy(TLS_last_native_error.msg_buff + pos, msg, len + 1); + pos += len; + + msg = ": "; + len = strlen(msg); + memcpy(TLS_last_native_error.msg_buff + pos, msg, len + 1); + pos += len; + + utils_strerror(TLS_last_native_error.errno_value, + TLS_last_native_error.msg_buff + pos, TLS_MSG_BUF_LEN - pos); + + *ppMessage = TLS_last_native_error.msg_buff; +} + +static umf_result_t fixed_get_recommended_page_size(void *provider, size_t size, + size_t *page_size) { + (void)provider; // unused + (void)size; // unused + + *page_size = utils_get_page_size(); + + return UMF_RESULT_SUCCESS; +} + +static umf_result_t fixed_get_min_page_size(void *provider, void *ptr, + size_t *page_size) { + (void)ptr; // unused + + return fixed_get_recommended_page_size(provider, 0, page_size); +} + +static umf_result_t fixed_purge_lazy(void *provider, void *ptr, size_t size) { + (void)provider; // unused + (void)ptr; // unused + (void)size; // unused + // purge_lazy is unsupported in case of the fixed memory provider + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + +static umf_result_t fixed_purge_force(void *provider, void *ptr, size_t size) { + (void)provider; // unused + errno = 0; + if (utils_purge(ptr, size, UMF_PURGE_FORCE)) { + fixed_store_last_native_error(UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED, + errno); + LOG_PERR("force purging failed"); + return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; + } + return UMF_RESULT_SUCCESS; +} + +static const char *fixed_get_name(void *provider) { + (void)provider; // unused + return "FIXED"; +} + +static umf_result_t fixed_allocation_split(void *provider, void *ptr, + size_t totalSize, size_t firstSize) { + fixed_memory_provider_t *fixed_provider = + (fixed_memory_provider_t *)provider; + return coarse_split(fixed_provider->coarse, ptr, totalSize, firstSize); +} + +static umf_result_t fixed_allocation_merge(void *provider, void *lowPtr, + void *highPtr, size_t totalSize) { + fixed_memory_provider_t *fixed_provider = + (fixed_memory_provider_t *)provider; + return coarse_merge(fixed_provider->coarse, lowPtr, highPtr, totalSize); +} + +static umf_result_t fixed_free(void *provider, void *ptr, size_t size) { + fixed_memory_provider_t *fixed_provider = + (fixed_memory_provider_t *)provider; + return coarse_free(fixed_provider->coarse, ptr, size); +} + +static umf_memory_provider_ops_t UMF_FIXED_MEMORY_PROVIDER_OPS = { + .version = UMF_VERSION_CURRENT, + .initialize = fixed_initialize, + .finalize = fixed_finalize, + .alloc = fixed_alloc, + .free = fixed_free, + .get_last_native_error = fixed_get_last_native_error, + .get_recommended_page_size = fixed_get_recommended_page_size, + .get_min_page_size = fixed_get_min_page_size, + .get_name = fixed_get_name, + .ext.purge_lazy = fixed_purge_lazy, + .ext.purge_force = fixed_purge_force, + .ext.allocation_merge = fixed_allocation_merge, + .ext.allocation_split = fixed_allocation_split, + .ipc.get_ipc_handle_size = NULL, + .ipc.get_ipc_handle = NULL, + .ipc.put_ipc_handle = NULL, + .ipc.open_ipc_handle = NULL, + .ipc.close_ipc_handle = NULL}; + +umf_memory_provider_ops_t *umfFixedMemoryProviderOps(void) { + return &UMF_FIXED_MEMORY_PROVIDER_OPS; +} + +umf_result_t umfFixedMemoryProviderParamsCreate( + umf_fixed_memory_provider_params_handle_t *hParams, void *ptr, + size_t size) { + libumfInit(); + if (hParams == NULL) { + LOG_ERR("Memory Provider params handle is NULL"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + umf_fixed_memory_provider_params_handle_t params = + umf_ba_global_alloc(sizeof(*params)); + if (params == NULL) { + LOG_ERR("Allocating memory for the Memory Provider params failed"); + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + umf_result_t ret = umfFixedMemoryProviderParamsSetMemory(params, ptr, size); + if (ret != UMF_RESULT_SUCCESS) { + umf_ba_global_free(params); + return ret; + } + + *hParams = params; + + return UMF_RESULT_SUCCESS; +} + +umf_result_t umfFixedMemoryProviderParamsDestroy( + umf_fixed_memory_provider_params_handle_t hParams) { + if (hParams != NULL) { + umf_ba_global_free(hParams); + } + + return UMF_RESULT_SUCCESS; +} + +umf_result_t umfFixedMemoryProviderParamsSetMemory( + umf_fixed_memory_provider_params_handle_t hParams, void *ptr, size_t size) { + + if (hParams == NULL) { + LOG_ERR("Memory Provider params handle is NULL"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (ptr == NULL) { + LOG_ERR("Memory pointer is NULL"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (size == 0) { + LOG_ERR("Size must be greater than 0"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + hParams->ptr = ptr; + hParams->size = size; + return UMF_RESULT_SUCCESS; +} diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index 5f9c85a86d..eaea8abd90 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -75,6 +75,14 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( return UMF_RESULT_ERROR_NOT_SUPPORTED; } +umf_result_t umfLevelZeroMemoryProviderParamsSetFreePolicy( + umf_level_zero_memory_provider_params_handle_t hParams, + umf_level_zero_memory_provider_free_policy_t policy) { + (void)hParams; + (void)policy; + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + umf_memory_provider_ops_t *umfLevelZeroMemoryProviderOps(void) { // not supported LOG_ERR("L0 memory provider is disabled! (UMF_BUILD_LEVEL_ZERO_PROVIDER is " @@ -107,6 +115,9 @@ typedef struct umf_level_zero_memory_provider_params_t { resident_device_handles; ///< Array of devices for which the memory should be made resident uint32_t resident_device_count; ///< Number of devices for which the memory should be made resident + + umf_level_zero_memory_provider_free_policy_t + freePolicy; ///< Memory free policy } umf_level_zero_memory_provider_params_t; typedef struct ze_memory_provider_t { @@ -118,6 +129,8 @@ typedef struct ze_memory_provider_t { uint32_t resident_device_count; ze_device_properties_t device_properties; + + ze_driver_memory_free_policy_ext_flags_t freePolicyFlags; } ze_memory_provider_t; typedef struct ze_ops_t { @@ -144,6 +157,8 @@ typedef struct ze_ops_t { size_t); ze_result_t (*zeDeviceGetProperties)(ze_device_handle_t, ze_device_properties_t *); + ze_result_t (*zeMemFreeExt)(ze_context_handle_t, + ze_memory_free_ext_desc_t *, void *); } ze_ops_t; static ze_ops_t g_ze_ops; @@ -197,6 +212,8 @@ static void init_ze_global_state(void) { utils_get_symbol_addr(0, "zeContextMakeMemoryResident", lib_name); *(void **)&g_ze_ops.zeDeviceGetProperties = utils_get_symbol_addr(0, "zeDeviceGetProperties", lib_name); + *(void **)&g_ze_ops.zeMemFreeExt = + utils_get_symbol_addr(0, "zeMemFreeExt", lib_name); if (!g_ze_ops.zeMemAllocHost || !g_ze_ops.zeMemAllocDevice || !g_ze_ops.zeMemAllocShared || !g_ze_ops.zeMemFree || @@ -232,6 +249,7 @@ umf_result_t umfLevelZeroMemoryProviderParamsCreate( params->memory_type = UMF_MEMORY_TYPE_UNKNOWN; params->resident_device_handles = NULL; params->resident_device_count = 0; + params->freePolicy = UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_DEFAULT; *hParams = params; @@ -308,6 +326,32 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( return UMF_RESULT_SUCCESS; } +umf_result_t umfLevelZeroMemoryProviderParamsSetFreePolicy( + umf_level_zero_memory_provider_params_handle_t hParams, + umf_level_zero_memory_provider_free_policy_t policy) { + if (!hParams) { + LOG_ERR("Level zero memory provider params handle is NULL"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + hParams->freePolicy = policy; + return UMF_RESULT_SUCCESS; +} + +static ze_driver_memory_free_policy_ext_flags_t +umfFreePolicyToZePolicy(umf_level_zero_memory_provider_free_policy_t policy) { + switch (policy) { + case UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_DEFAULT: + return 0; + case UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_BLOCKING_FREE: + return ZE_DRIVER_MEMORY_FREE_POLICY_EXT_FLAG_BLOCKING_FREE; + case UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_DEFER_FREE: + return ZE_DRIVER_MEMORY_FREE_POLICY_EXT_FLAG_DEFER_FREE; + default: + return 0; + } +} + static umf_result_t ze_memory_provider_initialize(void *params, void **provider) { if (params == NULL) { @@ -351,6 +395,12 @@ static umf_result_t ze_memory_provider_initialize(void *params, ze_provider->context = ze_params->level_zero_context_handle; ze_provider->device = ze_params->level_zero_device_handle; ze_provider->memory_type = (ze_memory_type_t)ze_params->memory_type; + ze_provider->freePolicyFlags = + umfFreePolicyToZePolicy(ze_params->freePolicy); + + memset(&ze_provider->device_properties, 0, + sizeof(ze_provider->device_properties)); + ze_provider->device_properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; if (ze_provider->device) { umf_result_t ret = ze2umf_result(g_ze_ops.zeDeviceGetProperties( @@ -361,9 +411,6 @@ static umf_result_t ze_memory_provider_initialize(void *params, umf_ba_global_free(ze_provider); return ret; } - } else { - memset(&ze_provider->device_properties, 0, - sizeof(ze_provider->device_properties)); } if (ze_params->resident_device_count) { @@ -492,8 +539,18 @@ static umf_result_t ze_memory_provider_free(void *provider, void *ptr, } ze_memory_provider_t *ze_provider = (ze_memory_provider_t *)provider; - ze_result_t ze_result = g_ze_ops.zeMemFree(ze_provider->context, ptr); - return ze2umf_result(ze_result); + + if (ze_provider->freePolicyFlags == 0) { + return ze2umf_result(g_ze_ops.zeMemFree(ze_provider->context, ptr)); + } + + ze_memory_free_ext_desc_t desc = { + .stype = ZE_STRUCTURE_TYPE_MEMORY_FREE_EXT_DESC, + .pNext = NULL, + .freePolicy = ze_provider->freePolicyFlags}; + + return ze2umf_result( + g_ze_ops.zeMemFreeExt(ze_provider->context, &desc, ptr)); } static void ze_memory_provider_get_last_native_error(void *provider, @@ -698,11 +755,11 @@ static struct umf_memory_provider_ops_t UMF_LEVEL_ZERO_MEMORY_PROVIDER_OPS = { .initialize = ze_memory_provider_initialize, .finalize = ze_memory_provider_finalize, .alloc = ze_memory_provider_alloc, + .free = ze_memory_provider_free, .get_last_native_error = ze_memory_provider_get_last_native_error, .get_recommended_page_size = ze_memory_provider_get_recommended_page_size, .get_min_page_size = ze_memory_provider_get_min_page_size, .get_name = ze_memory_provider_get_name, - .ext.free = ze_memory_provider_free, .ext.purge_lazy = ze_memory_provider_purge_lazy, .ext.purge_force = ze_memory_provider_purge_force, .ext.allocation_merge = ze_memory_provider_allocation_merge, diff --git a/src/provider/provider_os_memory.c b/src/provider/provider_os_memory.c index 4c19944a96..1fe467942f 100644 --- a/src/provider/provider_os_memory.c +++ b/src/provider/provider_os_memory.c @@ -1286,9 +1286,7 @@ static umf_result_t os_get_ipc_handle(void *provider, const void *ptr, void *value = critnib_get(os_provider->fd_offset_map, (uintptr_t)ptr); if (value == NULL) { - LOG_ERR("os_get_ipc_handle(): getting a value from the IPC cache " - "failed (addr=%p)", - ptr); + LOG_ERR("getting a value from the IPC cache failed (addr=%p)", ptr); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } @@ -1408,11 +1406,11 @@ static umf_memory_provider_ops_t UMF_OS_MEMORY_PROVIDER_OPS = { .initialize = os_initialize, .finalize = os_finalize, .alloc = os_alloc, + .free = os_free, .get_last_native_error = os_get_last_native_error, .get_recommended_page_size = os_get_recommended_page_size, .get_min_page_size = os_get_min_page_size, .get_name = os_get_name, - .ext.free = os_free, .ext.purge_lazy = os_purge_lazy, .ext.purge_force = os_purge_force, .ext.allocation_merge = os_allocation_merge, diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index e726feefb0..c4fff4133b 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -154,9 +154,6 @@ typedef struct umf_tracking_memory_provider_t { umf_memory_pool_handle_t pool; critnib *ipcCache; ipc_mapped_handle_cache_handle_t hIpcMappedCache; - - // the upstream provider does not support the free() operation - bool upstreamDoesNotFree; } umf_tracking_memory_provider_t; typedef struct umf_tracking_memory_provider_t umf_tracking_memory_provider_t; @@ -422,8 +419,7 @@ static umf_result_t trackingInitialize(void *params, void **ret) { // TODO clearing the tracker is a temporary solution and should be removed. // The tracker should be cleared using the provider's free() operation. static void clear_tracker_for_the_pool(umf_memory_tracker_handle_t hTracker, - umf_memory_pool_handle_t pool, - bool upstreamDoesNotFree) { + umf_memory_pool_handle_t pool) { uintptr_t rkey; void *rvalue; size_t n_items = 0; @@ -448,7 +444,7 @@ static void clear_tracker_for_the_pool(umf_memory_tracker_handle_t hTracker, #ifndef NDEBUG // print error messages only if provider supports the free() operation - if (n_items && !upstreamDoesNotFree) { + if (n_items) { if (pool) { LOG_ERR( "tracking provider of pool %p is not empty! (%zu items left)", @@ -459,13 +455,12 @@ static void clear_tracker_for_the_pool(umf_memory_tracker_handle_t hTracker, } } #else /* DEBUG */ - (void)upstreamDoesNotFree; // unused in DEBUG build - (void)n_items; // unused in DEBUG build + (void)n_items; // unused in DEBUG build #endif /* DEBUG */ } static void clear_tracker(umf_memory_tracker_handle_t hTracker) { - clear_tracker_for_the_pool(hTracker, NULL, false); + clear_tracker_for_the_pool(hTracker, NULL); } static void trackingFinalize(void *provider) { @@ -480,8 +475,7 @@ static void trackingFinalize(void *provider) { // because it may need those resources till // the very end of exiting the application. if (!utils_is_running_in_proxy_lib()) { - clear_tracker_for_the_pool(p->hTracker, p->pool, - p->upstreamDoesNotFree); + clear_tracker_for_the_pool(p->hTracker, p->pool); } umf_ba_global_free(provider); @@ -760,11 +754,11 @@ umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { .initialize = trackingInitialize, .finalize = trackingFinalize, .alloc = trackingAlloc, + .free = trackingFree, .get_last_native_error = trackingGetLastError, .get_min_page_size = trackingGetMinPageSize, .get_recommended_page_size = trackingGetRecommendedPageSize, .get_name = trackingName, - .ext.free = trackingFree, .ext.purge_force = trackingPurgeForce, .ext.purge_lazy = trackingPurgeLazy, .ext.allocation_split = trackingAllocationSplit, @@ -777,11 +771,10 @@ umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { umf_result_t umfTrackingMemoryProviderCreate( umf_memory_provider_handle_t hUpstream, umf_memory_pool_handle_t hPool, - umf_memory_provider_handle_t *hTrackingProvider, bool upstreamDoesNotFree) { + umf_memory_provider_handle_t *hTrackingProvider) { umf_tracking_memory_provider_t params; params.hUpstream = hUpstream; - params.upstreamDoesNotFree = upstreamDoesNotFree; params.hTracker = TRACKER; if (!params.hTracker) { LOG_ERR("failed, TRACKER is NULL"); diff --git a/src/provider/provider_tracking.h b/src/provider/provider_tracking.h index 9444ee4757..2abc365051 100644 --- a/src/provider/provider_tracking.h +++ b/src/provider/provider_tracking.h @@ -54,7 +54,7 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, // forwards all requests to hUpstream memory Provider. hUpstream lifetime should be managed by the user of this function. umf_result_t umfTrackingMemoryProviderCreate( umf_memory_provider_handle_t hUpstream, umf_memory_pool_handle_t hPool, - umf_memory_provider_handle_t *hTrackingProvider, bool upstreamDoesNotFree); + umf_memory_provider_handle_t *hTrackingProvider); void umfTrackingMemoryProviderGetUpstreamProvider( umf_memory_provider_handle_t hTrackingProvider, diff --git a/src/proxy_lib/proxy_lib.c b/src/proxy_lib/proxy_lib.c index f8bae304d8..15ddfca1bc 100644 --- a/src/proxy_lib/proxy_lib.c +++ b/src/proxy_lib/proxy_lib.c @@ -128,13 +128,6 @@ static umf_memory_pool_handle_t Proxy_pool = NULL; // it protects us from recursion in umfPool*() static __TLS int was_called_from_umfPool = 0; -// This WA for the issue: -// https://github.com/oneapi-src/unified-memory-framework/issues/894 -// It protects us from a recursion in malloc_usable_size() -// when the JEMALLOC proxy_lib_pool is used. -// TODO remove this WA when the issue is fixed. -static __TLS int was_called_from_malloc_usable_size = 0; - /*****************************************************************************/ /*** The constructor and destructor of the proxy library *********************/ /*****************************************************************************/ @@ -478,18 +471,15 @@ size_t malloc_usable_size(void *ptr) { return 0; // unsupported in case of the ba_leak allocator } - if (!was_called_from_malloc_usable_size && Proxy_pool && - (umfPoolByPtr(ptr) == Proxy_pool)) { - was_called_from_malloc_usable_size = 1; + if (Proxy_pool && (umfPoolByPtr(ptr) == Proxy_pool)) { was_called_from_umfPool = 1; size_t size = umfPoolMallocUsableSize(Proxy_pool, ptr); was_called_from_umfPool = 0; - was_called_from_malloc_usable_size = 0; return size; } #ifndef _WIN32 - if (!was_called_from_malloc_usable_size && Size_threshold_value) { + if (Size_threshold_value) { return System_malloc_usable_size(ptr); } #endif /* _WIN32 */ diff --git a/src/proxy_lib/proxy_lib.rc.in b/src/proxy_lib/proxy_lib.rc.in index dce151ec3e..f0497fb400 100644 --- a/src/proxy_lib/proxy_lib.rc.in +++ b/src/proxy_lib/proxy_lib.rc.in @@ -1,4 +1,4 @@ -// Copyright (c) 2024 Intel Corporation +// Copyright (C) 2024-2025 Intel Corporation // // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -51,7 +51,7 @@ BEGIN VALUE "CompanyName", "Intel Corporation\0" VALUE "FileDescription", "Unified Memory Framework (UMF) proxy library\0" VALUE "FileVersion", _UMF_VERSION "\0" - VALUE "LegalCopyright", "Copyright 2024, Intel Corporation. All rights reserved.\0" + VALUE "LegalCopyright", "Copyright 2024-2025, Intel Corporation. All rights reserved.\0" VALUE "LegalTrademarks", "\0" VALUE "OriginalFilename", "umf_proxy.dll\0" VALUE "ProductName", "Unified Memory Framework (UMF)\0" diff --git a/src/utils/utils_common.h b/src/utils/utils_common.h index 9ef2b3cf13..6af5a08d91 100644 --- a/src/utils/utils_common.h +++ b/src/utils/utils_common.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/utils/utils_posix_common.c b/src/utils/utils_posix_common.c index 4a60cbb1f2..613b8ea41d 100644 --- a/src/utils/utils_posix_common.c +++ b/src/utils/utils_posix_common.c @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -91,9 +91,8 @@ umf_result_t utils_duplicate_fd(int pid, int fd_in, int *fd_out) { return UMF_RESULT_ERROR_NOT_SUPPORTED; #else // pidfd_getfd(2) is used to obtain a duplicate of another process's file descriptor. - // Permission to duplicate another process's file descriptor - // is governed by a ptrace access mode PTRACE_MODE_ATTACH_REALCREDS check (see ptrace(2)) - // that can be changed using the /proc/sys/kernel/yama/ptrace_scope interface. + // Calling prctl(PR_SET_PTRACER, getppid()) in a producer binary that creates IPC handle + // allows file descriptor duplication for parent process and its children. // pidfd_getfd(2) is supported since Linux 5.6 // pidfd_open(2) is supported since Linux 5.3 errno = 0; diff --git a/src/utils/utils_posix_concurrency.c b/src/utils/utils_posix_concurrency.c index fcf04ed952..531e09c10a 100644 --- a/src/utils/utils_posix_concurrency.c +++ b/src/utils/utils_posix_concurrency.c @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/utils/utils_windows_concurrency.c b/src/utils/utils_windows_concurrency.c index 696f4523bc..e2cc574a9e 100644 --- a/src/utils/utils_windows_concurrency.c +++ b/src/utils/utils_windows_concurrency.c @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/utils/utils_windows_math.c b/src/utils/utils_windows_math.c index 07c4c9978b..cd21ae696f 100644 --- a/src/utils/utils_windows_math.c +++ b/src/utils/utils_windows_math.c @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c8b854ba5d..7eed07e09e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,10 +1,16 @@ -# Copyright (C) 2022-2024 Intel Corporation +# Copyright (C) 2022-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED YES) +if(CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") + # Compiler dependencies needs to be in library path or to be linked + # statically + add_link_options(-static-intel) +endif() + include(FetchContent) FetchContent_Declare( googletest @@ -41,14 +47,27 @@ function(build_umf_test) set(LIB_DIRS ${LIB_DIRS} ${LIBHWLOC_LIBRARY_DIRS}) - if(UMF_BUILD_LIBUMF_POOL_JEMALLOC) - set(LIB_DIRS ${LIB_DIRS} ${JEMALLOC_LIBRARY_DIRS}) - endif() - if(UMF_BUILD_CUDA_PROVIDER) + set(INC_DIRS ${INC_DIRS} ${CUDA_INCLUDE_DIRS}) set(LIB_DIRS ${LIB_DIRS} ${CUDA_LIBRARY_DIRS}) endif() + if(UMF_BUILD_LEVEL_ZERO_PROVIDER) + set(INC_DIRS ${INC_DIRS} ${LEVEL_ZERO_INCLUDE_DIRS}) + endif() + + if(UMF_POOL_JEMALLOC_ENABLED) + set(CPL_DEFS ${CPL_DEFS} UMF_POOL_JEMALLOC_ENABLED=1) + endif() + + if(UMF_POOL_SCALABLE_ENABLED) + set(CPL_DEFS ${CPL_DEFS} UMF_POOL_SCALABLE_ENABLED=1) + endif() + + if(UMF_BUILD_LIBUMF_POOL_DISJOINT) + set(CPL_DEFS ${CPL_DEFS} UMF_POOL_DISJOINT_ENABLED=1) + endif() + set(TEST_LIBS umf_test_common ${ARG_LIBS} @@ -61,15 +80,7 @@ function(build_umf_test) SRCS ${ARG_SRCS} LIBS ${TEST_LIBS}) - if(UMF_POOL_JEMALLOC_ENABLED) - target_compile_definitions(${TEST_TARGET_NAME} - PRIVATE UMF_POOL_JEMALLOC_ENABLED=1) - endif() - - if(UMF_POOL_SCALABLE_ENABLED) - target_compile_definitions(${TEST_TARGET_NAME} - PRIVATE UMF_POOL_SCALABLE_ENABLED=1) - endif() + target_compile_definitions(${TEST_TARGET_NAME} PRIVATE ${CPL_DEFS}) if(NOT MSVC) # Suppress 'cast discards const qualifier' warnings. Parametrized GTEST @@ -81,6 +92,7 @@ function(build_umf_test) target_compile_options(${TEST_TARGET_NAME} PRIVATE -Werror) endif() endif() + target_link_directories(${TEST_TARGET_NAME} PRIVATE ${LIB_DIRS}) target_include_directories( @@ -88,9 +100,11 @@ function(build_umf_test) PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include ${UMF_CMAKE_SOURCE_DIR}/src ${UMF_CMAKE_SOURCE_DIR}/src/base_alloc + ${UMF_CMAKE_SOURCE_DIR}/src/coarse ${UMF_CMAKE_SOURCE_DIR}/src/utils ${UMF_TEST_DIR}/common - ${UMF_TEST_DIR}) + ${UMF_TEST_DIR} + ${INC_DIRS}) endfunction() function(add_umf_test) @@ -156,8 +170,8 @@ if(UMF_BUILD_SHARED_LIBRARY) endif() endif() -if(UMF_POOL_JEMALLOC_ENABLED) - set(LIB_JEMALLOC_POOL jemalloc_pool) +if(UMF_BUILD_LIBUMF_POOL_DISJOINT) + set(LIB_DISJOINT_POOL disjoint_pool) endif() if(UMF_BUILD_SHARED_LIBRARY) @@ -179,6 +193,11 @@ add_umf_test( SRCS utils/utils_log.cpp ${UMF_UTILS_SOURCES} LIBS ${UMF_LOGGER_LIBS}) +add_umf_test( + NAME ctl + SRCS ctl/test.cpp ctl/ctl_debug.c ../src/ctl/ctl.c ${BA_SOURCES_FOR_TEST} + LIBS ${UMF_UTILS_FOR_TEST}) + add_umf_test( NAME utils_common SRCS utils/utils.cpp @@ -192,9 +211,9 @@ if(LINUX) endif() add_umf_test( - NAME provider_coarse - SRCS provider_coarse.cpp ${BA_SOURCES_FOR_TEST} - LIBS ${UMF_UTILS_FOR_TEST}) + NAME coarse_lib + SRCS coarse_lib.cpp ${BA_SOURCES_FOR_TEST} + LIBS ${UMF_UTILS_FOR_TEST} coarse) if(UMF_BUILD_LIBUMF_POOL_DISJOINT) add_umf_test( @@ -205,10 +224,13 @@ if(UMF_BUILD_LIBUMF_POOL_DISJOINT) NAME c_api_disjoint_pool SRCS c_api/disjoint_pool.c LIBS disjoint_pool) - add_umf_test( - NAME disjointCoarseMallocPool - SRCS disjointCoarseMallocPool.cpp - LIBS disjoint_pool) + if(LINUX AND (NOT UMF_DISABLE_HWLOC)) + # this test uses the file provider + add_umf_test( + NAME disjointPoolFileProv + SRCS disjointPoolFileProv.cpp + LIBS disjoint_pool) + endif() endif() if(UMF_BUILD_LIBUMF_POOL_DISJOINT @@ -218,14 +240,15 @@ if(UMF_BUILD_LIBUMF_POOL_DISJOINT add_umf_test( NAME c_api_multi_pool SRCS c_api/multi_pool.c - LIBS disjoint_pool jemalloc_pool ${JEMALLOC_LIBRARIES}) + LIBS disjoint_pool) endif() if(UMF_POOL_JEMALLOC_ENABLED AND (NOT UMF_DISABLE_HWLOC)) add_umf_test( NAME jemalloc_pool SRCS pools/jemalloc_pool.cpp malloc_compliance_tests.cpp - LIBS jemalloc_pool) + ${BA_SOURCES_FOR_TEST} + LIBS ${UMF_UTILS_FOR_TEST}) endif() if(UMF_POOL_SCALABLE_ENABLED AND (NOT UMF_DISABLE_HWLOC)) @@ -248,13 +271,7 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented add_umf_test( NAME provider_os_memory SRCS provider_os_memory.cpp ${BA_SOURCES_FOR_TEST} - LIBS ${UMF_UTILS_FOR_TEST} ${LIB_JEMALLOC_POOL}) - if(UMF_BUILD_LIBUMF_POOL_DISJOINT) - target_compile_definitions(umf_test-provider_os_memory - PRIVATE UMF_POOL_DISJOINT_ENABLED=1) - target_link_libraries(umf_test-provider_os_memory PRIVATE disjoint_pool) - endif() - + LIBS ${UMF_UTILS_FOR_TEST} ${LIB_DISJOINT_POOL}) add_umf_test( NAME provider_os_memory_multiple_numa_nodes SRCS provider_os_memory_multiple_numa_nodes.cpp @@ -302,7 +319,7 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented add_umf_test( NAME provider_devdax_memory_ipc SRCS provider_devdax_memory_ipc.cpp ${BA_SOURCES_FOR_TEST} - LIBS ${UMF_UTILS_FOR_TEST} ${LIB_JEMALLOC_POOL}) + LIBS ${UMF_UTILS_FOR_TEST}) add_umf_test( NAME provider_file_memory SRCS provider_file_memory.cpp @@ -310,18 +327,24 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented add_umf_test( NAME provider_file_memory_ipc SRCS provider_file_memory_ipc.cpp ${BA_SOURCES_FOR_TEST} - LIBS ${UMF_UTILS_FOR_TEST} ${LIB_JEMALLOC_POOL}) + LIBS ${UMF_UTILS_FOR_TEST}) + add_umf_test( + NAME provider_fixed_memory + SRCS provider_fixed_memory.cpp + LIBS ${UMF_UTILS_FOR_TEST}) # This test requires Linux-only file memory provider if(UMF_POOL_JEMALLOC_ENABLED) add_umf_test( NAME jemalloc_coarse_file SRCS pools/jemalloc_coarse_file.cpp malloc_compliance_tests.cpp - LIBS jemalloc_pool) + ${BA_SOURCES_FOR_TEST} + LIBS ${UMF_UTILS_FOR_TEST}) add_umf_test( NAME jemalloc_coarse_devdax SRCS pools/jemalloc_coarse_devdax.cpp malloc_compliance_tests.cpp - LIBS jemalloc_pool) + ${BA_SOURCES_FOR_TEST} + LIBS ${UMF_UTILS_FOR_TEST}) endif() # This test requires Linux-only file memory provider @@ -368,8 +391,6 @@ if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_LEVEL_ZERO_PROVIDER) SRCS providers/provider_level_zero.cpp ${UMF_UTILS_DIR}/utils_level_zero.cpp ${BA_SOURCES_FOR_TEST} LIBS ${UMF_UTILS_FOR_TEST} ze_loader) - target_include_directories(umf_test-provider_level_zero - PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS}) add_umf_test( NAME provider_level_zero_dlopen @@ -378,8 +399,6 @@ if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_LEVEL_ZERO_PROVIDER) LIBS ${UMF_UTILS_FOR_TEST}) target_compile_definitions(umf_test-provider_level_zero_dlopen PUBLIC USE_DLOPEN=1) - target_include_directories(umf_test-provider_level_zero_dlopen - PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS}) endif() if(NOT UMF_BUILD_LEVEL_ZERO_PROVIDER) @@ -399,10 +418,6 @@ if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_CUDA_PROVIDER) SRCS providers/provider_cuda.cpp providers/cuda_helpers.cpp ${BA_SOURCES_FOR_TEST} LIBS ${UMF_UTILS_FOR_TEST} cuda) - target_include_directories(umf_test-provider_cuda - PRIVATE ${CUDA_INCLUDE_DIRS}) - target_link_directories(umf_test-provider_cuda PRIVATE - ${CUDA_LIBRARY_DIRS}) add_umf_test( NAME provider_cuda_dlopen @@ -411,8 +426,6 @@ if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_CUDA_PROVIDER) LIBS ${UMF_UTILS_FOR_TEST}) target_compile_definitions(umf_test-provider_cuda_dlopen PUBLIC USE_DLOPEN=1) - target_include_directories(umf_test-provider_cuda_dlopen - PRIVATE ${CUDA_INCLUDE_DIRS}) else() message( STATUS @@ -453,10 +466,10 @@ if(UMF_PROXY_LIB_ENABLED AND UMF_BUILD_SHARED_LIBRARY) # TODO enable this test on Windows if(LINUX) add_umf_test( - NAME test_proxy_lib_size_threshold + NAME proxy_lib_size_threshold SRCS ${BA_SOURCES_FOR_TEST} test_proxy_lib_size_threshold.cpp LIBS ${UMF_UTILS_FOR_TEST} umf_proxy) - set_property(TEST umf-test_proxy_lib_size_threshold + set_property(TEST umf-proxy_lib_size_threshold PROPERTY ENVIRONMENT UMF_PROXY="size.threshold=64") endif() @@ -604,10 +617,6 @@ if(LINUX) ze_loader disjoint_pool ${UMF_UTILS_FOR_TEST}) - target_include_directories(umf_test-ipc_level_zero_prov_producer - PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS}) - target_include_directories(umf_test-ipc_level_zero_prov_consumer - PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS}) add_umf_ipc_test(TEST ipc_level_zero_prov SRC_DIR providers) endif() @@ -638,10 +647,6 @@ if(LINUX) cuda disjoint_pool ${UMF_UTILS_FOR_TEST}) - target_include_directories(umf_test-ipc_cuda_prov_producer - PRIVATE ${CUDA_INCLUDE_DIRS}) - target_include_directories(umf_test-ipc_cuda_prov_consumer - PRIVATE ${CUDA_INCLUDE_DIRS}) add_umf_ipc_test(TEST ipc_cuda_prov SRC_DIR providers) endif() else() @@ -745,7 +750,7 @@ if(LINUX else() message( STATUS - "The dram_and_fsdax example is supported on Linux only and requires UMF_BUILD_LIBUMF_POOL_JEMALLOC to be turned ON - skipping" + "The dram_and_fsdax example is supported on Linux only and requires the jemalloc pool, but it is disabled - skipping" ) endif() @@ -753,6 +758,13 @@ if(LINUX set(STANDALONE_CMAKE_OPTIONS "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" ) + if(JEMALLOC_INCLUDE_DIRS) + # add custom jemalloc installation + set(STANDALONE_CMAKE_OPTIONS + "${STANDALONE_CMAKE_OPTIONS} -DCMAKE_PREFIX_PATH=${JEMALLOC_INCLUDE_DIRS}/../" + ) + endif() + add_test( NAME umf-standalone_examples COMMAND diff --git a/test/coarse_lib.cpp b/test/coarse_lib.cpp new file mode 100644 index 0000000000..c5e30ee8fc --- /dev/null +++ b/test/coarse_lib.cpp @@ -0,0 +1,1351 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include "coarse.h" +#include "provider.hpp" + +using umf_test::KB; +using umf_test::MB; +using umf_test::test; + +#define MOCKED_COARSE ((coarse_t *)0x01) +#define MOCKED_PROVIDER ((umf_memory_provider_handle_t)0x02) +#define INVALID_PTR ((void *)0x03) + +static umf_result_t alloc_cb(void *provider, size_t size, size_t alignment, + void **ptr) { + return umfMemoryProviderAlloc((umf_memory_provider_handle_t)provider, size, + alignment, ptr); +} + +static umf_result_t free_cb(void *provider, void *ptr, size_t size) { + return umfMemoryProviderFree((umf_memory_provider_handle_t)provider, ptr, + size); +} + +static umf_result_t split_cb(void *provider, void *ptr, size_t totalSize, + size_t firstSize) { + if (provider == NULL || ptr == NULL || (firstSize >= totalSize) || + firstSize == 0 || totalSize == 0) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + return UMF_RESULT_SUCCESS; +} + +static umf_result_t merge_cb(void *provider, void *lowPtr, void *highPtr, + size_t totalSize) { + if (provider == NULL || lowPtr == NULL || highPtr == NULL || + totalSize == 0 || ((uintptr_t)highPtr <= (uintptr_t)lowPtr) || + ((uintptr_t)highPtr - (uintptr_t)lowPtr >= totalSize)) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + return UMF_RESULT_SUCCESS; +} + +static umf_result_t alloc_cb_fails(void *provider, size_t size, + size_t alignment, void **ptr) { + (void)provider; //unused + (void)size; //unused + (void)alignment; //unused + (void)ptr; //unused + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; +} + +static umf_result_t free_cb_fails(void *provider, void *ptr, size_t size) { + (void)provider; //unused + (void)ptr; //unused + (void)size; //unused + return UMF_RESULT_ERROR_USER_SPECIFIC; +} + +static umf_result_t split_cb_fails(void *provider, void *ptr, size_t totalSize, + size_t firstSize) { + (void)provider; //unused + (void)ptr; //unused + (void)totalSize; //unused + (void)firstSize; //unused + return UMF_RESULT_ERROR_USER_SPECIFIC; +} + +static umf_result_t merge_cb_fails(void *provider, void *lowPtr, void *highPtr, + size_t totalSize) { + (void)provider; //unused + (void)lowPtr; //unused + (void)highPtr; //unused + (void)totalSize; //unused + return UMF_RESULT_ERROR_USER_SPECIFIC; +} + +static void coarse_params_set_default(coarse_params_t *coarse_params, + umf_memory_provider_handle_t provider, + coarse_strategy_t allocation_strategy) { + memset(coarse_params, 0, sizeof(*coarse_params)); + coarse_params->provider = provider; + coarse_params->allocation_strategy = allocation_strategy; + coarse_params->cb.split = split_cb; + coarse_params->cb.merge = merge_cb; + coarse_params->page_size = utils_get_page_size(); + + if (provider) { + coarse_params->cb.alloc = alloc_cb; + coarse_params->cb.free = free_cb; + } +} + +umf_memory_provider_ops_t UMF_MALLOC_MEMORY_PROVIDER_OPS = + umf::providerMakeCOps(); + +struct CoarseWithMemoryStrategyTest + : umf_test::test, + ::testing::WithParamInterface { + void SetUp() override { + test::SetUp(); + allocation_strategy = this->GetParam(); + coarse_params_set_default(&coarse_params, MOCKED_PROVIDER, + allocation_strategy); + } + + coarse_t *coarse_handle = nullptr; + coarse_params_t coarse_params; + coarse_strategy_t allocation_strategy; + umf_result_t umf_result; +}; + +INSTANTIATE_TEST_SUITE_P( + CoarseWithMemoryStrategyTest, CoarseWithMemoryStrategyTest, + ::testing::Values(UMF_COARSE_MEMORY_STRATEGY_FASTEST, + UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE, + UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE)); + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_basic_provider) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + const size_t alloc_size = 20 * MB; + void *ptr; + + umf_result = coarse_add_memory_from_provider(ch, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + umf_result = coarse_free(ch, ptr, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + coarse_delete(ch); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_basic_fixed_memory) { + // preallocate some memory and initialize the vector with zeros + const size_t buff_size = 20 * MB + coarse_params.page_size; + std::vector buffer(buff_size, 0); + void *buf = (void *)ALIGN_UP_SAFE((uintptr_t)buffer.data(), + coarse_params.page_size); + ASSERT_NE(buf, nullptr); + + coarse_params.cb.alloc = NULL; + coarse_params.cb.free = NULL; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + char *ptr = nullptr; + + umf_result = coarse_add_memory_fixed(ch, buf, buff_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + umf_result = coarse_free(ch, ptr, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + coarse_delete(ch); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_fixed_memory_various) { + // preallocate some memory and initialize the vector with zeros + const size_t buff_size = 20 * MB + coarse_params.page_size; + std::vector buffer(buff_size, 0); + void *buf = (void *)ALIGN_UP_SAFE((uintptr_t)buffer.data(), + coarse_params.page_size); + ASSERT_NE(buf, nullptr); + + coarse_params.cb.alloc = NULL; + coarse_params.cb.free = NULL; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + char *ptr = nullptr; + + umf_result = coarse_add_memory_fixed(ch, buf, buff_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // free NULL + umf_result = coarse_free(ch, nullptr, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + // free invalid pointer + umf_result = coarse_free(ch, INVALID_PTR, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // wrong alignment (3 bytes) + ptr = nullptr; + umf_result = coarse_alloc(ch, 2 * MB, 3, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ALIGNMENT); + ASSERT_EQ(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // not freed allocation + // coarse_delete() prints LOG_WARN() in Debug mode + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + coarse_delete(ch); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_split_merge) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + char *ptr = nullptr; + const size_t alloc_size = 20 * MB; + + umf_result = coarse_add_memory_from_provider(ch, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + /* test coarse_split */ + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + umf_result = coarse_split(ch, ptr, 2 * MB, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + umf_result = coarse_free(ch, (ptr + 1 * MB), 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 1 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + umf_result = coarse_free(ch, ptr, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + /* test coarse_merge */ + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + umf_result = coarse_split(ch, ptr, 2 * MB, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + umf_result = coarse_merge(ch, ptr, (ptr + 1 * MB), 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + umf_result = coarse_free(ch, ptr, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + coarse_delete(coarse_handle); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +// negative tests + +// NULL parameters +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_no_params) { + umf_result = coarse_new(nullptr, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(coarse_handle, nullptr); +} + +// no provider +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_no_provider) { + coarse_params.provider = NULL; + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(coarse_handle, nullptr); +} + +// no page size +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_no_page_size) { + coarse_params.page_size = 0; + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(coarse_handle, nullptr); +} + +// no split callback +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_no_split_cb) { + coarse_params.cb.split = NULL; + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(coarse_handle, nullptr); +} + +// no merge callback +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_no_merge_cb) { + coarse_params.cb.merge = NULL; + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(coarse_handle, nullptr); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_alloc_invalid) { + void *ptr = nullptr; + + umf_result = coarse_alloc(nullptr, MB, 0, nullptr); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(ptr, nullptr); + + umf_result = coarse_alloc(nullptr, MB, 0, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(ptr, nullptr); + + umf_result = coarse_alloc(MOCKED_COARSE, MB, 0, nullptr); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(ptr, nullptr); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_free_invalid) { + // coarse handle is NULL + umf_result = coarse_free(nullptr, nullptr, MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_delete_null) { + coarse_delete(nullptr); +} + +TEST_P(CoarseWithMemoryStrategyTest, + coarseTest_add_memory_from_provider_null_0) { + umf_result = coarse_add_memory_from_provider(nullptr, 0); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_add_memory_fixed_null_0) { + umf_result = coarse_add_memory_fixed(nullptr, nullptr, 0); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_null_stats) { + ASSERT_EQ(coarse_get_stats(nullptr).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(nullptr).used_size, 0); + ASSERT_EQ(coarse_get_stats(nullptr).num_all_blocks, 0); + ASSERT_EQ(coarse_get_stats(nullptr).num_free_blocks, 0); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_split_merge_negative) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + char *ptr = nullptr; + const size_t alloc_size = 20 * MB; + + umf_result = coarse_add_memory_from_provider(ch, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + /* test coarse_split */ + + umf_result = coarse_alloc(ch, 6 * MB, 0, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 6 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + // firstSize >= totalSize + umf_result = coarse_split(ch, ptr, 6 * MB, 6 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // firstSize == 0 + umf_result = coarse_split(ch, ptr, 6 * MB, 0); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // totalSize == 0 + umf_result = coarse_split(ch, ptr, 0, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // wrong totalSize + umf_result = coarse_split(ch, ptr, 5 * MB, 1 * KB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // memory block not found + umf_result = coarse_split(ch, ptr + 1, 6 * MB, 1 * KB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + umf_result = coarse_free(ch, ptr, 6 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // split freed block + umf_result = coarse_split(ch, ptr, alloc_size, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + /* test coarse_merge */ + + umf_result = coarse_alloc(ch, 6 * MB, 0, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 6 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + // split (6 * MB) block into (1 * MB) + (5 * MB) + umf_result = coarse_split(ch, ptr, 6 * MB, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 6 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + // split (5 * MB) block into (2 * MB) + (3 * MB) + umf_result = coarse_split(ch, (ptr + 1 * MB), 5 * MB, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 6 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 4); + + // now we have 3 used blocks: (1 * MB) + (2 * MB) + (3 * MB) + + // highPtr <= lowPtr + umf_result = coarse_merge(ch, (ptr + 1 * MB), ptr, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // highPtr - lowPtr >= totalSize + umf_result = coarse_merge(ch, ptr, (ptr + 1 * MB), 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // low ptr does not exist + umf_result = coarse_merge(ch, ptr + 1, (ptr + 1 * MB), 3 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // high ptr does not exist + umf_result = coarse_merge(ch, ptr, (ptr + 1 * MB + 1), 3 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // low_block->size + high_block->size != totalSize + umf_result = coarse_merge(ch, ptr, (ptr + 1 * MB), 5 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // not adjacent blocks + umf_result = coarse_merge(ch, ptr, (ptr + 3 * MB), 4 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // free the 2 MB block in the middle + umf_result = coarse_free(ch, (ptr + 1 * MB), 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 4 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 4); + + // now we have 3 blocks: (1 * MB) used + (2 * MB) freed + (3 * MB) used + + // the low ptr block is not allocated + umf_result = coarse_merge(ch, (ptr + 1 * MB), (ptr + 3 * MB), 5 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // the high ptr block is not allocated + umf_result = coarse_merge(ch, ptr, (ptr + 1 * MB), 3 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + umf_result = coarse_free(ch, ptr, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 3 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + umf_result = coarse_free(ch, (ptr + 3 * MB), 3 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + coarse_delete(coarse_handle); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_basic_alloc_cb_fails) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + coarse_params.cb.alloc = alloc_cb_fails; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + const size_t alloc_size = 20 * MB; + + umf_result = coarse_add_memory_from_provider(ch, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + coarse_delete(ch); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_basic_free_cb_fails) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + coarse_params.cb.free = free_cb_fails; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + const size_t alloc_size = 20 * MB; + + umf_result = coarse_add_memory_from_provider(ch, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + coarse_delete(ch); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_split_cb_fails) { + if (coarse_params.allocation_strategy == + UMF_COARSE_MEMORY_STRATEGY_FASTEST) { + // This test is designed for the UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE + // and UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE strategies, + // because the UMF_COARSE_MEMORY_STRATEGY_FASTEST strategy + // looks always for a block of size greater by the page size. + return; + } + + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + coarse_params.cb.split = split_cb_fails; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + void *ptr = nullptr; + const size_t alloc_size = 20 * MB; + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + umf_result = coarse_add_memory_from_provider(ch, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // coarse_alloc(alloc_size / 2, alignment = 0) + umf_result = coarse_alloc(ch, alloc_size / 2, 0, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_USER_SPECIFIC); + ASSERT_EQ(ptr, nullptr); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // coarse_alloc(alloc_size / 2, alignment = 2 * MB) + umf_result = coarse_alloc(ch, alloc_size / 2, 2 * MB, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_USER_SPECIFIC); + ASSERT_EQ(ptr, nullptr); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // coarse_alloc(alloc_size, alignment = 0) - OK + umf_result = coarse_alloc(ch, alloc_size, 0, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + + ASSERT_EQ(coarse_get_stats(ch).used_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + umf_result = coarse_split(ch, ptr, alloc_size, alloc_size / 2); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_USER_SPECIFIC); + + ASSERT_EQ(coarse_get_stats(ch).used_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + umf_result = coarse_free(ch, ptr, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + coarse_delete(coarse_handle); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_merge_cb_fails) { + // preallocate some memory and initialize the vector with zeros + const size_t buff_size = 10 * MB + coarse_params.page_size; + std::vector buffer(buff_size, 0); + void *buf = (void *)ALIGN_UP_SAFE((uintptr_t)buffer.data(), + coarse_params.page_size); + ASSERT_NE(buf, nullptr); + + coarse_params.cb.alloc = NULL; + coarse_params.cb.free = NULL; + coarse_params.cb.merge = merge_cb_fails; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + char *ptr = nullptr; + + umf_result = coarse_add_memory_fixed(ch, buf, buff_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + /* test coarse_merge */ + umf_result = coarse_alloc(ch, 3 * MB, 0, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 3 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + umf_result = coarse_split(ch, ptr, 3 * MB, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 3 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + umf_result = coarse_merge(ch, ptr, (ptr + 1 * MB), 3 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_USER_SPECIFIC); + ASSERT_EQ(coarse_get_stats(ch).used_size, 3 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + umf_result = coarse_free(ch, ptr, 3 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(coarse_get_stats(ch).used_size, 3 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + umf_result = coarse_free(ch, ptr, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + umf_result = coarse_free(ch, (ptr + 1 * MB), 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + coarse_delete(coarse_handle); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_fixed_memory_alloc_set) { + // preallocate some memory and initialize the vector with zeros + const size_t buff_size = 20 * MB; + std::vector buffer(buff_size, 0); + void *buf = (void *)buffer.data(); + ASSERT_NE(buf, nullptr); + + coarse_params.cb.free = NULL; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + umf_result = coarse_add_memory_fixed(ch, buf, buff_size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + coarse_delete(ch); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_fixed_memory_free_set) { + // preallocate some memory and initialize the vector with zeros + const size_t buff_size = 20 * MB; + std::vector buffer(buff_size, 0); + void *buf = (void *)buffer.data(); + ASSERT_NE(buf, nullptr); + + coarse_params.cb.alloc = NULL; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + umf_result = coarse_add_memory_fixed(ch, buf, buff_size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + coarse_delete(ch); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_fixed_memory_alloc_free_set) { + // preallocate some memory and initialize the vector with zeros + const size_t buff_size = 20 * MB; + std::vector buffer(buff_size, 0); + void *buf = (void *)buffer.data(); + ASSERT_NE(buf, nullptr); + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + umf_result = coarse_add_memory_fixed(ch, buf, buff_size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + coarse_delete(ch); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_provider_alloc_not_set) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + coarse_params.cb.alloc = NULL; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + const size_t alloc_size = 20 * MB; + void *ptr; + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + umf_result = coarse_add_memory_from_provider(ch, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + umf_result = coarse_alloc(ch, 2 * MB, 0, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY); + ASSERT_EQ(ptr, nullptr); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + umf_result = coarse_alloc(ch, 2 * MB, 2 * MB, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY); + ASSERT_EQ(ptr, nullptr); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + coarse_delete(ch); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_basic) { + if (coarse_params.allocation_strategy == + UMF_COARSE_MEMORY_STRATEGY_FASTEST) { + // This test is designed for the UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE + // and UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE strategies, + // because the UMF_COARSE_MEMORY_STRATEGY_FASTEST strategy + // looks always for a block of size greater by the page size. + return; + } + + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + const size_t init_buffer_size = 20 * MB; + void *p1, *p2; + + umf_result = coarse_add_memory_from_provider(ch, init_buffer_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // alloc 2x 2MB + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&p1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(p1, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&p2); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(p2, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 4 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + ASSERT_NE(p1, p2); + + // swap pointers to get p1 < p2 + if (p1 > p2) { + std::swap(p1, p2); + } + + // free + alloc first block + // the block should be reused + // currently there is no purging, so the alloc size shouldn't change + // there should be no block merging between used and not-used blocks + umf_result = coarse_free(ch, p1, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&p1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(p1, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 4 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + // free all allocs + // overall alloc size shouldn't change + // block p2 should merge with the prev free block p1 + // and the remaining init block + umf_result = coarse_free(ch, p1, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + umf_result = coarse_free(ch, p2, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // test allocations with alignment + // TODO: what about holes? + umf_result = coarse_alloc(ch, 1 * MB - 4, 128, (void **)&p1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(p1, nullptr); + ASSERT_EQ((uintptr_t)p1 & 127, 0); + + umf_result = coarse_alloc(ch, 1 * MB - 4, 128, (void **)&p2); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(p2, nullptr); + ASSERT_EQ((uintptr_t)p2 & 127, 0); + + umf_result = coarse_free(ch, p1, 1 * MB - 4); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = coarse_free(ch, p2, 1 * MB - 4); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + // alloc whole buffer + // after this, there should be one single block + umf_result = coarse_alloc(ch, init_buffer_size, 0, (void **)&p1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(p1, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // free all memory + umf_result = coarse_free(ch, p1, init_buffer_size); + + // alloc 2 MB block - the init block should be split + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&p1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(p1, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + // alloc additional 2 MB + // the non-used block should be used + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&p2); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(p2, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 4 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + ASSERT_NE(p1, p2); + + // make sure that p1 < p2 + if (p1 > p2) { + std::swap(p1, p2); + } + + // free blocks in order: p2, p1 + // block p1 should merge with the next block p2 + // swap pointers to get p1 < p2 + coarse_free(ch, p2, 2 * MB); + coarse_free(ch, p1, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // alloc 10x 2 MB - this should occupy all allocated memory + constexpr int allocs_size = 10; + void *allocs[allocs_size] = {0}; + for (int i = 0; i < allocs_size; i++) { + ASSERT_EQ(coarse_get_stats(ch).used_size, i * 2 * MB); + umf_result = coarse_alloc(ch, 2 * MB, 0, &allocs[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(allocs[i], nullptr); + } + ASSERT_EQ(coarse_get_stats(ch).used_size, 20 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + // there should be no block with the free memory + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, allocs_size); + + // free all memory + for (int i = 0; i < allocs_size; i++) { + umf_result = coarse_free(ch, allocs[i], 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + + coarse_delete(ch); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_simple1) { + if (coarse_params.allocation_strategy == + UMF_COARSE_MEMORY_STRATEGY_FASTEST) { + // This test is designed for the UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE + // and UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE strategies, + // because the UMF_COARSE_MEMORY_STRATEGY_FASTEST strategy + // looks always for a block of size greater by the page size. + return; + } + + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + const size_t init_buffer_size = 20 * MB; + + umf_result = coarse_add_memory_from_provider(ch, init_buffer_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // test 1 + + size_t s1 = 74659 * KB; + size_t s2 = 8206 * KB; + + size_t max_alloc_size = 0; + + const int nreps = 2; + const int nptrs = 6; + + // s1 + for (int j = 0; j < nreps; j++) { + void *t[nptrs] = {0}; + for (int i = 0; i < nptrs; i++) { + umf_result = coarse_alloc(ch, s1, 0, &t[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(t[i], nullptr); + } + + size_t alloc_size = coarse_get_stats(ch).alloc_size; + if (alloc_size > max_alloc_size) { + max_alloc_size = alloc_size; + } + + for (int i = 0; i < nptrs; i++) { + umf_result = coarse_free(ch, t[i], s1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + } + + // s2 + for (int j = 0; j < nreps; j++) { + void *t[nptrs] = {0}; + for (int i = 0; i < nptrs; i++) { + umf_result = coarse_alloc(ch, s2, 0, &t[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(t[i], nullptr); + } + + // all s2 should fit into single block leaved after freeing s1 + ASSERT_LE(coarse_get_stats(ch).alloc_size, max_alloc_size); + + for (int i = 0; i < nptrs; i++) { + umf_result = coarse_free(ch, t[i], s2); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + } + + coarse_delete(ch); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_simple2) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + const size_t init_buffer_size = 20 * MB; + + umf_result = coarse_add_memory_from_provider(ch, init_buffer_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // test + double sizes[] = {2, 4, 0.5, 1, 8, 0.25}; + size_t alignment[] = {0, 4, 0, 16, 32, 128}; + for (int i = 0; i < 6; i++) { + size_t s = (size_t)(sizes[i] * MB); + void *t[8] = {0}; + for (int j = 0; j < 8; j++) { + umf_result = coarse_alloc(ch, s, alignment[i], &t[j]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(t[j], nullptr); + } + + for (int j = 0; j < 8; j++) { + umf_result = coarse_free(ch, t[j], s); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + } + + coarse_delete(ch); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_alignment_provider) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + const size_t alloc_size = 40 * MB; + + umf_result = coarse_add_memory_from_provider(ch, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + const int niter = 10; + const int size = 1 * MB; + void *ptr[niter] = {0}; + + for (int i = 0; i < niter; i++) { + umf_result = coarse_alloc(ch, size, 0, &ptr[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr[i], nullptr); + } + + ASSERT_EQ(coarse_get_stats(ch).used_size, niter * size); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, niter + 1); + + for (int i = 0; i < niter; i += 2) { + umf_result = coarse_free(ch, ptr[i], size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ptr[i] = nullptr; + } + + ASSERT_EQ(coarse_get_stats(ch).used_size, niter * size / 2); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, niter + 1); + + for (int i = 0; i < niter; i += 2) { + ASSERT_EQ(ptr[i], nullptr); + umf_result = coarse_alloc(ch, size, 2 * MB, &ptr[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr[i], nullptr); + } + + ASSERT_EQ(coarse_get_stats(ch).used_size, niter * size); + + for (int i = 0; i < niter; i++) { + umf_result = coarse_free(ch, ptr[i], size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + coarse_delete(ch); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_alignment_fixed_memory) { + // preallocate some memory and initialize the vector with zeros + const size_t alloc_size = 40 * MB + coarse_params.page_size; + std::vector buffer(alloc_size, 0); + void *buf = (void *)ALIGN_UP_SAFE((uintptr_t)buffer.data(), + coarse_params.page_size); + ASSERT_NE(buf, nullptr); + + coarse_params.cb.alloc = NULL; + coarse_params.cb.free = NULL; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + + umf_result = coarse_add_memory_fixed(ch, buf, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + const int niter = 10; + const int size = 1 * MB; + void *ptr[niter] = {0}; + + for (int i = 0; i < niter; i++) { + umf_result = coarse_alloc(ch, size, 0, &ptr[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr[i], nullptr); + } + + ASSERT_EQ(coarse_get_stats(ch).used_size, niter * size); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, niter + 1); + + for (int i = 0; i < niter; i += 2) { + umf_result = coarse_free(ch, ptr[i], size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ptr[i] = nullptr; + } + + ASSERT_EQ(coarse_get_stats(ch).used_size, niter * size / 2); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, niter + 1); + + for (int i = 0; i < niter; i += 2) { + ASSERT_EQ(ptr[i], nullptr); + umf_result = coarse_alloc(ch, size, 2 * MB, &ptr[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr[i], nullptr); + } + + ASSERT_EQ(coarse_get_stats(ch).used_size, niter * size); + + for (int i = 0; i < niter; i++) { + umf_result = coarse_free(ch, ptr[i], size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + coarse_delete(ch); +} diff --git a/test/common/ipc_common.c b/test/common/ipc_common.c index 140927079b..1590dd3c45 100644 --- a/test/common/ipc_common.c +++ b/test/common/ipc_common.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -336,6 +337,12 @@ int run_producer(int port, umf_memory_pool_ops_t *pool_ops, void *pool_params, int producer_socket = -1; char consumer_message[MSG_SIZE]; + ret = prctl(PR_SET_PTRACER, getppid()); + if (ret == -1) { + perror("PR_SET_PTRACER may be not supported. prctl() call failed"); + goto err_end; + } + // create OS memory provider umf_result = umfMemoryProviderCreate(provider_ops, provider_params, &provider); @@ -528,6 +535,7 @@ int run_producer(int port, umf_memory_pool_ops_t *pool_ops, void *pool_params, err_umfMemoryProviderDestroy: umfMemoryProviderDestroy(provider); +err_end: if (ret == 0) { fprintf(stderr, "[producer] Shutting down (status OK) ...\n"); } else if (ret == 1) { diff --git a/test/common/pool_trace.c b/test/common/pool_trace.c index 29329f31c0..d8b7522ea8 100644 --- a/test/common/pool_trace.c +++ b/test/common/pool_trace.c @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/test/common/provider_null.c b/test/common/provider_null.c index 5db389e895..e667bfce40 100644 --- a/test/common/provider_null.c +++ b/test/common/provider_null.c @@ -134,11 +134,11 @@ umf_memory_provider_ops_t UMF_NULL_PROVIDER_OPS = { .initialize = nullInitialize, .finalize = nullFinalize, .alloc = nullAlloc, + .free = nullFree, .get_last_native_error = nullGetLastError, .get_recommended_page_size = nullGetRecommendedPageSize, .get_min_page_size = nullGetPageSize, .get_name = nullName, - .ext.free = nullFree, .ext.purge_lazy = nullPurgeLazy, .ext.purge_force = nullPurgeForce, .ext.allocation_merge = nullAllocationMerge, diff --git a/test/common/provider_trace.c b/test/common/provider_trace.c index 219dde5cd7..9d063b4f56 100644 --- a/test/common/provider_trace.c +++ b/test/common/provider_trace.c @@ -195,11 +195,11 @@ umf_memory_provider_ops_t UMF_TRACE_PROVIDER_OPS = { .initialize = traceInitialize, .finalize = traceFinalize, .alloc = traceAlloc, + .free = traceFree, .get_last_native_error = traceGetLastError, .get_recommended_page_size = traceGetRecommendedPageSize, .get_min_page_size = traceGetPageSize, .get_name = traceName, - .ext.free = traceFree, .ext.purge_lazy = tracePurgeLazy, .ext.purge_force = tracePurgeForce, .ext.allocation_merge = traceAllocationMerge, diff --git a/test/common/test_helpers.c b/test/common/test_helpers.c index 71f018d0f7..d69ca35353 100644 --- a/test/common/test_helpers.c +++ b/test/common/test_helpers.c @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // This file contains tests for UMF pool API diff --git a/test/ctl/config.txt b/test/ctl/config.txt new file mode 100644 index 0000000000..5d4f9c62bd --- /dev/null +++ b/test/ctl/config.txt @@ -0,0 +1 @@ +debug.heap.alloc_pattern=321 \ No newline at end of file diff --git a/test/ctl/ctl_debug.c b/test/ctl/ctl_debug.c new file mode 100644 index 0000000000..711cb5e179 --- /dev/null +++ b/test/ctl/ctl_debug.c @@ -0,0 +1,126 @@ +/* + * + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +/* + * ctl_debug.c -- implementation of the debug CTL namespace + */ + +#include "ctl_debug.h" + +static struct ctl *ctl_debug; + +static int alloc_pattern = 0; +static int enable_logging = 0; +static int log_level = 0; + +struct ctl *get_debug_ctl(void) { return ctl_debug; } + +/* + * CTL_WRITE_HANDLER(alloc_pattern) -- sets the alloc_pattern field in heap + */ +static int CTL_WRITE_HANDLER(alloc_pattern)(void *ctx, + enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { + /* suppress unused-parameter errors */ + (void)source, (void)indexes, (void)ctx; + + int arg_in = *(int *)arg; + alloc_pattern = arg_in; + return 0; +} + +/* + * CTL_READ_HANDLER(alloc_pattern) -- returns alloc_pattern heap field + */ +static int CTL_READ_HANDLER(alloc_pattern)(void *ctx, + enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { + /* suppress unused-parameter errors */ + (void)source, (void)indexes, (void)ctx; + + int *arg_out = arg; + *arg_out = alloc_pattern; + return 0; +} + +static int CTL_WRITE_HANDLER(enable_logging)(void *ctx, + enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { + /* suppress unused-parameter errors */ + (void)source, (void)indexes, (void)ctx; + + int arg_in = *(int *)arg; + enable_logging = arg_in; + return 0; +} + +static int CTL_READ_HANDLER(enable_logging)(void *ctx, + enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { + /* suppress unused-parameter errors */ + (void)source, (void)indexes, (void)ctx; + + int *arg_out = arg; + *arg_out = enable_logging; + return 0; +} + +static int CTL_WRITE_HANDLER(log_level)(void *ctx, enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { + /* suppress unused-parameter errors */ + (void)source, (void)indexes, (void)ctx; + + int arg_in = *(int *)arg; + log_level = arg_in; + return 0; +} + +static int CTL_READ_HANDLER(log_level)(void *ctx, enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { + /* suppress unused-parameter errors */ + (void)source, (void)indexes, (void)ctx; + + int *arg_out = arg; + *arg_out = log_level; + return 0; +} + +static const struct ctl_argument CTL_ARG(alloc_pattern) = CTL_ARG_LONG_LONG; + +static const struct ctl_argument CTL_ARG(enable_logging) = CTL_ARG_BOOLEAN; + +static const struct ctl_argument CTL_ARG(log_level) = CTL_ARG_INT; + +static const struct ctl_node CTL_NODE(heap)[] = {CTL_LEAF_RW(alloc_pattern), + CTL_LEAF_RW(enable_logging), + CTL_LEAF_RW(log_level), + + CTL_NODE_END}; + +static const struct ctl_node CTL_NODE(debug)[] = {CTL_CHILD(heap), + + CTL_NODE_END}; + +/* + * debug_ctl_register -- registers ctl nodes for "debug" module + */ +void debug_ctl_register(struct ctl *ctl) { CTL_REGISTER_MODULE(ctl, debug); } + +void initialize_debug_ctl(void) { + ctl_debug = ctl_new(); + debug_ctl_register(ctl_debug); +} + +void deinitialize_debug_ctl(void) { ctl_delete(ctl_debug); } diff --git a/test/ctl/ctl_debug.h b/test/ctl/ctl_debug.h new file mode 100644 index 0000000000..9dd8bade5b --- /dev/null +++ b/test/ctl/ctl_debug.h @@ -0,0 +1,32 @@ +/* + * + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +/* + * ctl_debug.h -- definitions for CTL test + */ + +#ifndef UMF_CTL_DEBUG_H +#define UMF_CTL_DEBUG_H 1 + +#include "../src/ctl/ctl.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void debug_ctl_register(struct ctl *ctl); +struct ctl *get_debug_ctl(void); +void initialize_debug_ctl(void); +void deinitialize_debug_ctl(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/test/ctl/test.cpp b/test/ctl/test.cpp new file mode 100644 index 0000000000..c35759c673 --- /dev/null +++ b/test/ctl/test.cpp @@ -0,0 +1,93 @@ +/* + * + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#include "../common/base.hpp" +#include "ctl/ctl.h" +#include "ctl/ctl_debug.h" + +using namespace umf_test; + +TEST_F(test, ctl_debug_read_from_string) { + initialize_debug_ctl(); + auto ctl_handler = get_debug_ctl(); + ctl_load_config_from_string(ctl_handler, NULL, + "debug.heap.alloc_pattern=1"); + + int value = 0; + ctl_query(ctl_handler, NULL, CTL_QUERY_PROGRAMMATIC, + "debug.heap.alloc_pattern", CTL_QUERY_READ, &value); + ASSERT_EQ(value, 1); + + // Test setting alloc_pattern to 2 + ctl_load_config_from_string(ctl_handler, NULL, + "debug.heap.alloc_pattern=2"); + ctl_query(ctl_handler, NULL, CTL_QUERY_PROGRAMMATIC, + "debug.heap.alloc_pattern", CTL_QUERY_READ, &value); + ASSERT_EQ(value, 2); + + // Test setting alloc_pattern to 0 + ctl_load_config_from_string(ctl_handler, NULL, + "debug.heap.alloc_pattern=0"); + ctl_query(ctl_handler, NULL, CTL_QUERY_PROGRAMMATIC, + "debug.heap.alloc_pattern", CTL_QUERY_READ, &value); + ASSERT_EQ(value, 0); + + // Negative test: non-existent configuration + ASSERT_NE(ctl_query(ctl_handler, NULL, CTL_QUERY_PROGRAMMATIC, + "debug.heap.non_existent", CTL_QUERY_READ, &value), + 0); + + // Negative test: invalid path + ASSERT_NE(ctl_query(ctl_handler, NULL, CTL_QUERY_PROGRAMMATIC, + "invalid.path.alloc_pattern", CTL_QUERY_READ, &value), + 0); + + debug_ctl_register(ctl_handler); + deinitialize_debug_ctl(); +} + +int ctl_config_write_to_file(const char *filename, const char *data) { + FILE *file = fopen(filename == NULL ? "config.txt" : filename, "w+"); + if (file == NULL) { + return -1; + } + fputs(data, file); + fclose(file); + return 0; +} + +TEST_F(test, ctl_debug_read_from_file) { +#ifndef _WIN32 + ASSERT_EQ(ctl_config_write_to_file( + "config.txt", "debug.heap.alloc_pattern=321;\ndebug.heap." + "enable_logging=1;\ndebug.heap.log_level=5;\n"), + 0); + initialize_debug_ctl(); + auto ctl_handler = get_debug_ctl(); + ASSERT_EQ(ctl_load_config_from_file(ctl_handler, NULL, "config.txt"), 0); + + int value = 0; + ctl_query(ctl_handler, NULL, CTL_QUERY_PROGRAMMATIC, + "debug.heap.alloc_pattern", CTL_QUERY_READ, &value); + ASSERT_EQ(value, 321); + + value = 0; + ctl_query(ctl_handler, NULL, CTL_QUERY_PROGRAMMATIC, "debug.heap.log_level", + CTL_QUERY_READ, &value); + ASSERT_EQ(value, 5); + + value = 0; + ctl_query(ctl_handler, NULL, CTL_QUERY_PROGRAMMATIC, + "debug.heap.enable_logging", CTL_QUERY_READ, &value); + ASSERT_EQ(value, 1); + + debug_ctl_register(ctl_handler); + deinitialize_debug_ctl(); +#endif +} diff --git a/test/disjointCoarseMallocPool.cpp b/test/disjointCoarseMallocPool.cpp deleted file mode 100644 index 32e1d24f36..0000000000 --- a/test/disjointCoarseMallocPool.cpp +++ /dev/null @@ -1,580 +0,0 @@ -/* - * Copyright (C) 2023-2024 Intel Corporation - * - * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -*/ - -#include - -#include "provider.hpp" - -#include -#include - -using umf_test::KB; -using umf_test::MB; -using umf_test::test; - -#define GetStats umfCoarseMemoryProviderGetStats - -umf_memory_provider_ops_t UMF_MALLOC_MEMORY_PROVIDER_OPS = - umf::providerMakeCOps(); - -struct CoarseWithMemoryStrategyTest - : umf_test::test, - ::testing::WithParamInterface { - void SetUp() override { - test::SetUp(); - allocation_strategy = this->GetParam(); - } - - coarse_memory_provider_strategy_t allocation_strategy; -}; - -INSTANTIATE_TEST_SUITE_P( - CoarseWithMemoryStrategyTest, CoarseWithMemoryStrategyTest, - ::testing::Values(UMF_COARSE_MEMORY_STRATEGY_FASTEST, - UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE, - UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE)); - -TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMallocPool_basic) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - const size_t init_buffer_size = 20 * MB; - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.destroy_upstream_memory_provider = true; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = nullptr; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - umf_disjoint_pool_params_handle_t disjoint_pool_params = NULL; - umf_result = umfDisjointPoolParamsCreate(&disjoint_pool_params); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(disjoint_pool_params, nullptr); - umf_result = - umfDisjointPoolParamsSetSlabMinSize(disjoint_pool_params, 4096); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = - umfDisjointPoolParamsSetMaxPoolableSize(disjoint_pool_params, 4096); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = umfDisjointPoolParamsSetCapacity(disjoint_pool_params, 4); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = - umfDisjointPoolParamsSetMinBucketSize(disjoint_pool_params, 64); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = umfDisjointPoolParamsSetTrace(disjoint_pool_params, 1); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - - umf_memory_pool_handle_t pool; - umf_result = umfPoolCreate(umfDisjointPoolOps(), coarse_memory_provider, - disjoint_pool_params, - UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &pool); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(pool, nullptr); - - umf_result = umfDisjointPoolParamsDestroy(disjoint_pool_params); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - - // test - - umf_memory_provider_handle_t prov = NULL; - umf_result = umfPoolGetMemoryProvider(pool, &prov); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(prov, nullptr); - - // alloc 2x 2MB - void *p1 = umfPoolMalloc(pool, 2 * MB); - ASSERT_NE(p1, nullptr); - ASSERT_EQ(GetStats(prov).used_size, 2 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 2); - - void *p2 = umfPoolMalloc(pool, 2 * MB); - ASSERT_NE(p2, nullptr); - ASSERT_EQ(GetStats(prov).used_size, 4 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 3); - ASSERT_NE(p1, p2); - - // swap pointers to get p1 < p2 - if (p1 > p2) { - std::swap(p1, p2); - } - - // free + alloc first block - // the block should be reused - // currently there is no purging, so the alloc size shouldn't change - // there should be no block merging between used and not-used blocks - umf_result = umfPoolFree(pool, p1); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(prov).used_size, 2 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 3); - - p1 = umfPoolMalloc(pool, 2 * MB); - ASSERT_EQ(GetStats(prov).used_size, 4 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 3); - - // free all allocs - // overall alloc size shouldn't change - // block p2 should merge with the prev free block p1 - // and the remaining init block - umf_result = umfPoolFree(pool, p1); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(prov).num_all_blocks, 3); - umf_result = umfPoolFree(pool, p2); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(prov).used_size, 0 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 1); - - // test allocations with alignment - // TODO: what about holes? - p1 = umfPoolAlignedMalloc(pool, 1 * MB - 4, 128); - ASSERT_NE(p1, nullptr); - ASSERT_EQ((uintptr_t)p1 & 127, 0); - p2 = umfPoolAlignedMalloc(pool, 1 * MB - 4, 128); - ASSERT_NE(p2, nullptr); - ASSERT_EQ((uintptr_t)p1 & 127, 0); - umf_result = umfPoolFree(pool, p1); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = umfPoolFree(pool, p2); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - - // alloc whole buffer - // after this, there should be one single block - p1 = umfPoolMalloc(pool, init_buffer_size); - ASSERT_EQ(GetStats(prov).used_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 1); - - // free all memory - // alloc 2 MB block - the init block should be split - umf_result = umfPoolFree(pool, p1); - p1 = umfPoolMalloc(pool, 2 * MB); - ASSERT_EQ(GetStats(prov).used_size, 2 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 2); - - // alloc additional 2 MB - // the non-used block should be used - p2 = umfPoolMalloc(pool, 2 * MB); - ASSERT_EQ(GetStats(prov).used_size, 4 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 3); - ASSERT_NE(p1, p2); - - // make sure that p1 < p2 - if (p1 > p2) { - std::swap(p1, p2); - } - - // free blocks in order: p2, p1 - // block p1 should merge with the next block p2 - // swap pointers to get p1 < p2 - umfPoolFree(pool, p2); - umfPoolFree(pool, p1); - ASSERT_EQ(GetStats(prov).used_size, 0 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 1); - - // alloc 10x 2 MB - this should occupy all allocated memory - constexpr int allocs_size = 10; - void *allocs[allocs_size] = {0}; - for (int i = 0; i < allocs_size; i++) { - ASSERT_EQ(GetStats(prov).used_size, i * 2 * MB); - allocs[i] = umfPoolMalloc(pool, 2 * MB); - ASSERT_NE(allocs[i], nullptr); - } - ASSERT_EQ(GetStats(prov).used_size, 20 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - // there should be no block with the free memory - ASSERT_EQ(GetStats(prov).num_all_blocks, allocs_size); - - // free all memory - for (int i = 0; i < allocs_size; i++) { - umf_result = umfPoolFree(pool, allocs[i]); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - } - - ASSERT_EQ(GetStats(prov).num_all_blocks, 1); - ASSERT_EQ(GetStats(prov).used_size, 0 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - - umfPoolDestroy(pool); - // Both coarse_memory_provider and malloc_memory_provider - // have already been destroyed by umfPoolDestroy(), because: - // UMF_POOL_CREATE_FLAG_OWN_PROVIDER was set in umfPoolCreate() and - // coarse_memory_provider_params.destroy_upstream_memory_provider = true; -} - -TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMallocPool_simple1) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - const size_t init_buffer_size = 20 * MB; - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = NULL; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - umf_disjoint_pool_params_handle_t disjoint_pool_params = NULL; - umf_result = umfDisjointPoolParamsCreate(&disjoint_pool_params); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(disjoint_pool_params, nullptr); - umf_result = - umfDisjointPoolParamsSetSlabMinSize(disjoint_pool_params, 4096); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = - umfDisjointPoolParamsSetMaxPoolableSize(disjoint_pool_params, 4096); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = umfDisjointPoolParamsSetCapacity(disjoint_pool_params, 4); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = - umfDisjointPoolParamsSetMinBucketSize(disjoint_pool_params, 64); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = umfDisjointPoolParamsSetTrace(disjoint_pool_params, 1); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - - umf_memory_pool_handle_t pool; - umf_result = - umfPoolCreate(umfDisjointPoolOps(), coarse_memory_provider, - disjoint_pool_params, UMF_POOL_CREATE_FLAG_NONE, &pool); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(pool, nullptr); - - umf_result = umfDisjointPoolParamsDestroy(disjoint_pool_params); - - umf_memory_provider_handle_t prov = NULL; - umfPoolGetMemoryProvider(pool, &prov); - ASSERT_NE(prov, nullptr); - - // test 1 - - size_t s1 = 74659 * KB; - size_t s2 = 8206 * KB; - - size_t max_alloc_size = 0; - - const int nreps = 2; - const int nptrs = 6; - - // s1 - for (int j = 0; j < nreps; j++) { - void *t[nptrs] = {0}; - for (int i = 0; i < nptrs; i++) { - t[i] = umfPoolMalloc(pool, s1); - ASSERT_NE(t[i], nullptr); - } - - if (max_alloc_size == 0) { - max_alloc_size = GetStats(prov).alloc_size; - } - - for (int i = 0; i < nptrs; i++) { - umf_result = umfPoolFree(pool, t[i]); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - } - } - - // s2 - for (int j = 0; j < nreps; j++) { - void *t[nptrs] = {0}; - for (int i = 0; i < nptrs; i++) { - t[i] = umfPoolMalloc(pool, s2); - ASSERT_NE(t[i], nullptr); - } - - // all s2 should fit into single block leaved after freeing s1 - ASSERT_LE(GetStats(prov).alloc_size, max_alloc_size); - - for (int i = 0; i < nptrs; i++) { - umf_result = umfPoolFree(pool, t[i]); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - } - } - - umfPoolDestroy(pool); - umfMemoryProviderDestroy(coarse_memory_provider); - umfMemoryProviderDestroy(malloc_memory_provider); -} - -TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMallocPool_simple2) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - const size_t init_buffer_size = 20 * MB; - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = NULL; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - umf_disjoint_pool_params_handle_t disjoint_pool_params = NULL; - umf_result = umfDisjointPoolParamsCreate(&disjoint_pool_params); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(disjoint_pool_params, nullptr); - umf_result = - umfDisjointPoolParamsSetSlabMinSize(disjoint_pool_params, 4096); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = - umfDisjointPoolParamsSetMaxPoolableSize(disjoint_pool_params, 4096); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = umfDisjointPoolParamsSetCapacity(disjoint_pool_params, 4); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = - umfDisjointPoolParamsSetMinBucketSize(disjoint_pool_params, 64); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = umfDisjointPoolParamsSetTrace(disjoint_pool_params, 1); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - - umf_memory_pool_handle_t pool; - umf_result = - umfPoolCreate(umfDisjointPoolOps(), coarse_memory_provider, - disjoint_pool_params, UMF_POOL_CREATE_FLAG_NONE, &pool); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(pool, nullptr); - - umf_result = umfDisjointPoolParamsDestroy(disjoint_pool_params); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - - // test - double sizes[] = {2, 4, 0.5, 1, 8, 0.25}; - size_t alignment[] = {0, 4, 0, 16, 32, 128}; - for (int i = 0; i < 6; i++) { - size_t s = (size_t)(sizes[i] * MB); - void *t[8] = {0}; - for (int j = 0; j < 8; j++) { - t[j] = umfPoolAlignedMalloc(pool, s, alignment[i]); - ASSERT_NE(t[j], nullptr); - } - - for (int j = 0; j < 8; j++) { - umf_result = umfPoolFree(pool, t[j]); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - } - } - - umfPoolDestroy(pool); - umfMemoryProviderDestroy(coarse_memory_provider); - umfMemoryProviderDestroy(malloc_memory_provider); -} - -struct alloc_ptr_size { - void *ptr; - size_t size; - - bool operator<(const alloc_ptr_size &other) const { - if (ptr == other.ptr) { - return size < other.size; - } - return ptr < other.ptr; - } -}; - -TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMMapPool_random) { - umf_result_t umf_result; - - const size_t init_buffer_size = 200 * MB; - - // preallocate some memory and initialize the vector with zeros - std::vector buffer(init_buffer_size, 0); - void *buf = (void *)buffer.data(); - ASSERT_NE(buf, nullptr); - - const unsigned char alloc_check_val = 11; - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = NULL; - coarse_memory_provider_params.immediate_init_from_upstream = false; - coarse_memory_provider_params.init_buffer = buf; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - umf_disjoint_pool_params_handle_t disjoint_pool_params = NULL; - umf_result = umfDisjointPoolParamsCreate(&disjoint_pool_params); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(disjoint_pool_params, nullptr); - umf_result = - umfDisjointPoolParamsSetSlabMinSize(disjoint_pool_params, 1024); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = - umfDisjointPoolParamsSetMaxPoolableSize(disjoint_pool_params, 1024); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = umfDisjointPoolParamsSetCapacity(disjoint_pool_params, 2); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = - umfDisjointPoolParamsSetMinBucketSize(disjoint_pool_params, 16); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = umfDisjointPoolParamsSetTrace(disjoint_pool_params, 1); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - - umf_memory_pool_handle_t pool; - umf_result = - umfPoolCreate(umfDisjointPoolOps(), coarse_memory_provider, - disjoint_pool_params, UMF_POOL_CREATE_FLAG_NONE, &pool); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(pool, nullptr); - - umf_result = umfDisjointPoolParamsDestroy(disjoint_pool_params); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - - // set constant seed so each test run will have the same scenario - uint32_t seed = 1234; - std::mt19937 mt(seed); - - // different sizes to alloc - std::vector sizes = {15, 49, 588, 1025, - 2 * KB, 5 * KB, 160 * KB, 511 * KB, - 1000 * KB, MB, 3 * MB, 7 * MB}; - std::uniform_int_distribution sizes_dist(0, (int)(sizes.size() - 1)); - - // each alloc would be done few times - std::vector counts = {1, 3, 4, 8, 9, 11}; - std::uniform_int_distribution counts_dist(0, (int)(counts.size() - 1)); - - // action to take will be random - // alloc = <0, .5), free = <.5, 1) - std::uniform_real_distribution actions_dist(0, 1); - - std::set allocs; - const int nreps = 100; - - for (size_t i = 0; i < nreps; i++) { - size_t count = counts[counts_dist(mt)]; - float action = actions_dist(mt); - - if (action < 0.5) { - size_t size = sizes[sizes_dist(mt)]; - std::cout << "size: " << size << " count: " << count - << " action: alloc" << std::endl; - - // alloc - for (size_t j = 0; j < count; j++) { - void *ptr = umfPoolMalloc(pool, size); - ASSERT_NE(ptr, nullptr); - - if (ptr == nullptr) { - break; - } - - // check if first and last bytes are empty and fill them with control data - ASSERT_EQ(((unsigned char *)ptr)[0], 0); - ASSERT_EQ(((unsigned char *)ptr)[size - 1], 0); - ((unsigned char *)ptr)[0] = alloc_check_val; - ((unsigned char *)ptr)[size - 1] = alloc_check_val; - - allocs.insert({ptr, size}); - } - } else { - std::cout << "count: " << count << " action: free" << std::endl; - - // free random allocs - for (size_t j = 0; j < count; j++) { - if (allocs.size() == 0) { - continue; - } - - std::uniform_int_distribution free_dist( - 0, (int)(allocs.size() - 1)); - size_t free_id = free_dist(mt); - auto it = allocs.begin(); - std::advance(it, free_id); - auto [ptr, size] = (*it); - ASSERT_NE(ptr, nullptr); - - // check if control bytes are set and clean them - - ASSERT_EQ(((unsigned char *)ptr)[0], alloc_check_val); - ASSERT_EQ(((unsigned char *)ptr)[size - 1], alloc_check_val); - ((unsigned char *)ptr)[0] = 0; - ((unsigned char *)ptr)[size - 1] = 0; - - umf_result_t ret = umfPoolFree(pool, ptr); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - - allocs.erase(it); - } - } - } - - std::cout << "cleanup" << std::endl; - - while (allocs.size()) { - umf_result_t ret = umfPoolFree(pool, (*allocs.begin()).ptr); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - allocs.erase(allocs.begin()); - } - - umfPoolDestroy(pool); - umfMemoryProviderDestroy(coarse_memory_provider); -} diff --git a/test/disjointPoolFileProv.cpp b/test/disjointPoolFileProv.cpp new file mode 100644 index 0000000000..383487a87e --- /dev/null +++ b/test/disjointPoolFileProv.cpp @@ -0,0 +1,363 @@ +/* + * Copyright (C) 2023-2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include + +#include +#include + +#include "coarse.h" +#include "provider.hpp" + +using umf_test::KB; +using umf_test::MB; +using umf_test::test; + +#define FILE_PATH ((char *)"tmp_file") + +umf_memory_provider_ops_t UMF_MALLOC_MEMORY_PROVIDER_OPS = + umf::providerMakeCOps(); + +struct FileWithMemoryStrategyTest + : umf_test::test, + ::testing::WithParamInterface { + void SetUp() override { + test::SetUp(); + allocation_strategy = this->GetParam(); + } + + coarse_strategy_t allocation_strategy; +}; + +INSTANTIATE_TEST_SUITE_P( + FileWithMemoryStrategyTest, FileWithMemoryStrategyTest, + ::testing::Values(UMF_COARSE_MEMORY_STRATEGY_FASTEST, + UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE, + UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE)); + +TEST_P(FileWithMemoryStrategyTest, disjointFileMallocPool_simple1) { + umf_memory_provider_handle_t malloc_memory_provider = nullptr; + umf_result_t umf_result; + + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, + nullptr, &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + umf_file_memory_provider_params_handle_t file_params = nullptr; + umf_result = umfFileMemoryProviderParamsCreate(&file_params, FILE_PATH); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(file_params, nullptr); + + umf_memory_provider_handle_t file_memory_provider; + umf_result = umfMemoryProviderCreate(umfFileMemoryProviderOps(), + file_params, &file_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(file_memory_provider, nullptr); + + umf_result = umfFileMemoryProviderParamsDestroy(file_params); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_disjoint_pool_params_handle_t disjoint_pool_params = nullptr; + umf_result = umfDisjointPoolParamsCreate(&disjoint_pool_params); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(disjoint_pool_params, nullptr); + umf_result = + umfDisjointPoolParamsSetSlabMinSize(disjoint_pool_params, 4096); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = + umfDisjointPoolParamsSetMaxPoolableSize(disjoint_pool_params, 4096); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = umfDisjointPoolParamsSetCapacity(disjoint_pool_params, 4); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = + umfDisjointPoolParamsSetMinBucketSize(disjoint_pool_params, 64); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = umfDisjointPoolParamsSetTrace(disjoint_pool_params, 1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_memory_pool_handle_t pool; + umf_result = + umfPoolCreate(umfDisjointPoolOps(), file_memory_provider, + disjoint_pool_params, UMF_POOL_CREATE_FLAG_NONE, &pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(pool, nullptr); + + umf_result = umfDisjointPoolParamsDestroy(disjoint_pool_params); + + umf_memory_provider_handle_t prov = nullptr; + umfPoolGetMemoryProvider(pool, &prov); + ASSERT_NE(prov, nullptr); + + // test 1 + + size_t s1 = 74659 * KB; + size_t s2 = 8206 * KB; + + const int nreps = 2; + const int nptrs = 6; + + // s1 + for (int j = 0; j < nreps; j++) { + void *t[nptrs] = {0}; + for (int i = 0; i < nptrs; i++) { + t[i] = umfPoolMalloc(pool, s1); + ASSERT_NE(t[i], nullptr); + } + + for (int i = 0; i < nptrs; i++) { + umf_result = umfPoolFree(pool, t[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + } + + // s2 + for (int j = 0; j < nreps; j++) { + void *t[nptrs] = {0}; + for (int i = 0; i < nptrs; i++) { + t[i] = umfPoolMalloc(pool, s2); + ASSERT_NE(t[i], nullptr); + } + + for (int i = 0; i < nptrs; i++) { + umf_result = umfPoolFree(pool, t[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + } + + umfPoolDestroy(pool); + umfMemoryProviderDestroy(file_memory_provider); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(FileWithMemoryStrategyTest, disjointFileMallocPool_simple2) { + umf_memory_provider_handle_t malloc_memory_provider = nullptr; + umf_result_t umf_result; + + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, + nullptr, &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + umf_file_memory_provider_params_handle_t file_params = nullptr; + umf_result = umfFileMemoryProviderParamsCreate(&file_params, FILE_PATH); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(file_params, nullptr); + + umf_memory_provider_handle_t file_memory_provider; + umf_result = umfMemoryProviderCreate(umfFileMemoryProviderOps(), + file_params, &file_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(file_memory_provider, nullptr); + + umf_result = umfFileMemoryProviderParamsDestroy(file_params); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_disjoint_pool_params_handle_t disjoint_pool_params = nullptr; + umf_result = umfDisjointPoolParamsCreate(&disjoint_pool_params); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(disjoint_pool_params, nullptr); + umf_result = + umfDisjointPoolParamsSetSlabMinSize(disjoint_pool_params, 4096); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = + umfDisjointPoolParamsSetMaxPoolableSize(disjoint_pool_params, 4096); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = umfDisjointPoolParamsSetCapacity(disjoint_pool_params, 4); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = + umfDisjointPoolParamsSetMinBucketSize(disjoint_pool_params, 64); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = umfDisjointPoolParamsSetTrace(disjoint_pool_params, 1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_memory_pool_handle_t pool; + umf_result = + umfPoolCreate(umfDisjointPoolOps(), file_memory_provider, + disjoint_pool_params, UMF_POOL_CREATE_FLAG_NONE, &pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(pool, nullptr); + + umf_result = umfDisjointPoolParamsDestroy(disjoint_pool_params); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + // test + double sizes[] = {2, 4, 0.5, 1, 8, 0.25}; + size_t alignment[] = {0, 4, 0, 16, 32, 128}; + for (int i = 0; i < 6; i++) { + size_t s = (size_t)(sizes[i] * MB); + void *t[8] = {0}; + for (int j = 0; j < 8; j++) { + t[j] = umfPoolAlignedMalloc(pool, s, alignment[i]); + ASSERT_NE(t[j], nullptr); + } + + for (int j = 0; j < 8; j++) { + umf_result = umfPoolFree(pool, t[j]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + } + + umfPoolDestroy(pool); + umfMemoryProviderDestroy(file_memory_provider); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +struct alloc_ptr_size { + void *ptr; + size_t size; + + bool operator<(const alloc_ptr_size &other) const { + if (ptr == other.ptr) { + return size < other.size; + } + return ptr < other.ptr; + } +}; + +TEST_P(FileWithMemoryStrategyTest, disjointFileMMapPool_random) { + umf_result_t umf_result; + + const size_t init_buffer_size = 200 * MB; + + // preallocate some memory and initialize the vector with zeros + std::vector buffer(init_buffer_size, 0); + void *buf = (void *)buffer.data(); + ASSERT_NE(buf, nullptr); + + const unsigned char alloc_check_val = 11; + + umf_file_memory_provider_params_handle_t file_params = nullptr; + umf_result = umfFileMemoryProviderParamsCreate(&file_params, FILE_PATH); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(file_params, nullptr); + + umf_memory_provider_handle_t file_memory_provider; + umf_result = umfMemoryProviderCreate(umfFileMemoryProviderOps(), + file_params, &file_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(file_memory_provider, nullptr); + + umf_result = umfFileMemoryProviderParamsDestroy(file_params); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_disjoint_pool_params_handle_t disjoint_pool_params = nullptr; + umf_result = umfDisjointPoolParamsCreate(&disjoint_pool_params); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(disjoint_pool_params, nullptr); + umf_result = + umfDisjointPoolParamsSetSlabMinSize(disjoint_pool_params, 1024); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = + umfDisjointPoolParamsSetMaxPoolableSize(disjoint_pool_params, 1024); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = umfDisjointPoolParamsSetCapacity(disjoint_pool_params, 2); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = + umfDisjointPoolParamsSetMinBucketSize(disjoint_pool_params, 16); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = umfDisjointPoolParamsSetTrace(disjoint_pool_params, 1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_memory_pool_handle_t pool; + umf_result = + umfPoolCreate(umfDisjointPoolOps(), file_memory_provider, + disjoint_pool_params, UMF_POOL_CREATE_FLAG_NONE, &pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(pool, nullptr); + + umf_result = umfDisjointPoolParamsDestroy(disjoint_pool_params); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + // set constant seed so each test run will have the same scenario + uint32_t seed = 1234; + std::mt19937 mt(seed); + + // different sizes to alloc + std::vector sizes = {15, 49, 588, 1025, + 2 * KB, 5 * KB, 160 * KB, 511 * KB, + 1000 * KB, MB, 3 * MB, 7 * MB}; + std::uniform_int_distribution sizes_dist(0, (int)(sizes.size() - 1)); + + // each alloc would be done few times + std::vector counts = {1, 3, 4, 8, 9, 11}; + std::uniform_int_distribution counts_dist(0, (int)(counts.size() - 1)); + + // action to take will be random + // alloc = <0, .5), free = <.5, 1) + std::uniform_real_distribution actions_dist(0, 1); + + std::set allocs; + const int nreps = 100; + + for (size_t i = 0; i < nreps; i++) { + size_t count = counts[counts_dist(mt)]; + float action = actions_dist(mt); + + if (action < 0.5) { + size_t size = sizes[sizes_dist(mt)]; + std::cout << "size: " << size << " count: " << count + << " action: alloc" << std::endl; + + // alloc + for (size_t j = 0; j < count; j++) { + void *ptr = umfPoolCalloc(pool, 1, size); + if (ptr == nullptr) { + break; + } + + // check if first and last bytes are empty and fill them with control data + ASSERT_EQ(((unsigned char *)ptr)[0], 0); + ASSERT_EQ(((unsigned char *)ptr)[size - 1], 0); + ((unsigned char *)ptr)[0] = alloc_check_val; + ((unsigned char *)ptr)[size - 1] = alloc_check_val; + + allocs.insert({ptr, size}); + } + } else { + std::cout << "count: " << count << " action: free" << std::endl; + + // free random allocs + for (size_t j = 0; j < count; j++) { + if (allocs.size() == 0) { + continue; + } + + std::uniform_int_distribution free_dist( + 0, (int)(allocs.size() - 1)); + size_t free_id = free_dist(mt); + auto it = allocs.begin(); + std::advance(it, free_id); + auto [ptr, size] = (*it); + ASSERT_NE(ptr, nullptr); + + // check if control bytes are set and clean them + + ASSERT_EQ(((unsigned char *)ptr)[0], alloc_check_val); + ASSERT_EQ(((unsigned char *)ptr)[size - 1], alloc_check_val); + ((unsigned char *)ptr)[0] = 0; + ((unsigned char *)ptr)[size - 1] = 0; + + umf_result_t ret = umfPoolFree(pool, ptr); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + allocs.erase(it); + } + } + } + + std::cout << "cleanup" << std::endl; + + while (allocs.size()) { + umf_result_t ret = umfPoolFree(pool, (*allocs.begin()).ptr); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + allocs.erase(allocs.begin()); + } + + umfPoolDestroy(pool); + umfMemoryProviderDestroy(file_memory_provider); +} diff --git a/test/ipcAPI.cpp b/test/ipcAPI.cpp index 4df32a1c9b..aa22f353dc 100644 --- a/test/ipcAPI.cpp +++ b/test/ipcAPI.cpp @@ -116,4 +116,4 @@ HostMemoryAccessor hostMemoryAccessor; INSTANTIATE_TEST_SUITE_P(umfIpcTestSuite, umfIpcTest, ::testing::Values(ipcTestParams{ umfProxyPoolOps(), nullptr, &IPC_MOCK_PROVIDER_OPS, - nullptr, &hostMemoryAccessor, false})); + nullptr, &hostMemoryAccessor})); diff --git a/test/ipcFixtures.hpp b/test/ipcFixtures.hpp index 8dca83f10e..fd5260d1be 100644 --- a/test/ipcFixtures.hpp +++ b/test/ipcFixtures.hpp @@ -47,25 +47,23 @@ class HostMemoryAccessor : public MemoryAccessor { }; // ipcTestParams: -// pool_ops, pool_params, provider_ops, provider_params, memoryAccessor, free_not_supp -// free_not_supp (bool) - provider does not support the free() op +// pool_ops, pool_params, provider_ops, provider_params, memoryAccessor using ipcTestParams = std::tuple; + void *, MemoryAccessor *>; struct umfIpcTest : umf_test::test, ::testing::WithParamInterface { umfIpcTest() {} void SetUp() override { test::SetUp(); - auto [pool_ops, pool_params, provider_ops, provider_params, accessor, - free_not_supp] = this->GetParam(); + auto [pool_ops, pool_params, provider_ops, provider_params, accessor] = + this->GetParam(); poolOps = pool_ops; poolParams = pool_params; providerOps = provider_ops; providerParams = provider_params; memAccessor = accessor; - freeNotSupported = free_not_supp; } void TearDown() override { test::TearDown(); } @@ -124,18 +122,8 @@ struct umfIpcTest : umf_test::test, void *poolParams = nullptr; umf_memory_provider_ops_t *providerOps = nullptr; void *providerParams = nullptr; - bool freeNotSupported = false; }; -static inline umf_result_t -get_umf_result_of_free(bool freeNotSupported, umf_result_t expected_result) { - if (freeNotSupported) { - return UMF_RESULT_ERROR_NOT_SUPPORTED; - } - - return expected_result; -} - TEST_P(umfIpcTest, GetIPCHandleSize) { size_t size = 0; umf::pool_unique_handle_t pool = makePool(); @@ -177,8 +165,7 @@ TEST_P(umfIpcTest, GetIPCHandleInvalidArgs) { EXPECT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); ret = umfFree(ptr); - EXPECT_EQ(ret, - get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); } TEST_P(umfIpcTest, CloseIPCHandleInvalidPtr) { @@ -244,8 +231,7 @@ TEST_P(umfIpcTest, BasicFlow) { EXPECT_EQ(ret, UMF_RESULT_SUCCESS); ret = umfPoolFree(pool.get(), ptr); - EXPECT_EQ(ret, - get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); pool.reset(nullptr); EXPECT_EQ(stat.getCount, 1); @@ -313,8 +299,7 @@ TEST_P(umfIpcTest, GetPoolByOpenedHandle) { for (size_t i = 0; i < NUM_ALLOCS; ++i) { umf_result_t ret = umfFree(ptrs[i]); - EXPECT_EQ(ret, - get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); } } @@ -346,8 +331,7 @@ TEST_P(umfIpcTest, AllocFreeAllocTest) { EXPECT_EQ(ret, UMF_RESULT_SUCCESS); ret = umfPoolFree(pool.get(), ptr); - EXPECT_EQ(ret, - get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); ptr = umfPoolMalloc(pool.get(), SIZE); ASSERT_NE(ptr, nullptr); @@ -369,8 +353,7 @@ TEST_P(umfIpcTest, AllocFreeAllocTest) { EXPECT_EQ(ret, UMF_RESULT_SUCCESS); ret = umfPoolFree(pool.get(), ptr); - EXPECT_EQ(ret, - get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); pool.reset(nullptr); EXPECT_EQ(stat.getCount, stat.putCount); @@ -432,8 +415,7 @@ TEST_P(umfIpcTest, openInTwoIpcHandlers) { EXPECT_EQ(ret, UMF_RESULT_SUCCESS); ret = umfPoolFree(pool1.get(), ptr); - EXPECT_EQ(ret, - get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); pool1.reset(nullptr); pool2.reset(nullptr); @@ -484,8 +466,7 @@ TEST_P(umfIpcTest, ConcurrentGetPutHandles) { for (void *ptr : ptrs) { umf_result_t ret = umfPoolFree(pool.get(), ptr); - EXPECT_EQ(ret, - get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); } pool.reset(nullptr); @@ -552,8 +533,7 @@ TEST_P(umfIpcTest, ConcurrentOpenCloseHandles) { for (void *ptr : ptrs) { ret = umfPoolFree(pool.get(), ptr); - EXPECT_EQ(ret, - get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); } pool.reset(nullptr); diff --git a/test/ipc_os_prov_anon_fd.sh b/test/ipc_os_prov_anon_fd.sh index c5738e9893..a42d820a25 100755 --- a/test/ipc_os_prov_anon_fd.sh +++ b/test/ipc_os_prov_anon_fd.sh @@ -1,5 +1,5 @@ # -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2024-2025 Intel Corporation # # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -12,21 +12,6 @@ set -e # port should be a number from the range <1024, 65535> PORT=$(( 1024 + ( $$ % ( 65535 - 1024 )))) -# The ipc_os_prov_anon_fd example requires using pidfd_getfd(2) -# to obtain a duplicate of another process's file descriptor. -# Permission to duplicate another process's file descriptor -# is governed by a ptrace access mode PTRACE_MODE_ATTACH_REALCREDS check (see ptrace(2)) -# that can be changed using the /proc/sys/kernel/yama/ptrace_scope interface. -PTRACE_SCOPE_FILE="/proc/sys/kernel/yama/ptrace_scope" -VAL=0 -if [ -f $PTRACE_SCOPE_FILE ]; then - PTRACE_SCOPE_VAL=$(cat $PTRACE_SCOPE_FILE) - if [ $PTRACE_SCOPE_VAL -ne $VAL ]; then - echo "SKIP: ptrace_scope is not set to 0 (classic ptrace permissions) - skipping the test" - exit 125 # skip code defined in CMakeLists.txt - fi -fi - UMF_LOG_VAL="level:debug;flush:debug;output:stderr;pid:yes" echo "Starting ipc_os_prov_anon_fd CONSUMER on port $PORT ..." diff --git a/test/malloc_compliance_tests.cpp b/test/malloc_compliance_tests.cpp index 06e3b5dd78..b91bde1f6d 100644 --- a/test/malloc_compliance_tests.cpp +++ b/test/malloc_compliance_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/test/memoryPoolAPI.cpp b/test/memoryPoolAPI.cpp index 1c6d83f2af..0a45bfbe53 100644 --- a/test/memoryPoolAPI.cpp +++ b/test/memoryPoolAPI.cpp @@ -178,19 +178,132 @@ TEST_F(test, BasicPoolByPtrTest) { ASSERT_EQ(ret, UMF_RESULT_SUCCESS); } +struct tagTest : umf_test::test { + void SetUp() override { + test::SetUp(); + provider = umf_test::wrapProviderUnique(nullProviderCreate()); + pool = umf_test::wrapPoolUnique( + createPoolChecked(umfProxyPoolOps(), provider.get(), nullptr)); + } + + umf::provider_unique_handle_t provider; + umf::pool_unique_handle_t pool; +}; + +TEST_F(tagTest, SetAndGet) { + umf_result_t ret = umfPoolSetTag(pool.get(), (void *)0x99, nullptr); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + void *tag; + ret = umfPoolGetTag(pool.get(), &tag); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_EQ(tag, (void *)0x99); + + void *oldTag; + ret = umfPoolSetTag(pool.get(), (void *)0x100, &oldTag); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_EQ(oldTag, (void *)0x99); + + ret = umfPoolGetTag(pool.get(), &tag); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_EQ(tag, (void *)0x100); +} + +TEST_F(tagTest, SetAndGetNull) { + umf_result_t ret = umfPoolSetTag(pool.get(), nullptr, nullptr); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + void *tag; + ret = umfPoolGetTag(pool.get(), &tag); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_EQ(tag, nullptr); +} + +TEST_F(tagTest, NoSetAndGet) { + void *tag; + umf_result_t ret = umfPoolGetTag(pool.get(), &tag); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_EQ(tag, nullptr); +} + +TEST_F(tagTest, SetAndGetMt) { + static constexpr size_t NUM_THREADS = 8; + static constexpr size_t NUM_OPS_PER_THREAD = 16; + + std::vector threads; + + auto encodeTag = [](size_t thread, size_t op) -> void * { + return reinterpret_cast(thread * NUM_OPS_PER_THREAD + op); + }; + + auto decodeTag = [](void *tag) -> std::pair { + auto op = reinterpret_cast(tag) & (NUM_OPS_PER_THREAD - 1); + auto thread = reinterpret_cast(tag) / NUM_OPS_PER_THREAD; + return {thread, op}; + }; + + for (size_t i = 0; i < NUM_THREADS; i++) { + threads.emplace_back([this, i, encodeTag, decodeTag] { + for (size_t j = 0; j < NUM_OPS_PER_THREAD; j++) { + void *oldTag; + umf_result_t ret = + umfPoolSetTag(pool.get(), encodeTag(i, j), &oldTag); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + void *queriedTag; + ret = umfPoolGetTag(pool.get(), &queriedTag); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + auto [t1, op1] = decodeTag(oldTag); + auto [t2, op2] = decodeTag(queriedTag); + // if the tag was set by the same thread, the op part should be the same or higher + ASSERT_TRUE(t1 != t2 || op2 >= op1); + } + }); + } + + for (auto &thread : threads) { + thread.join(); + } + + void *tag; + auto ret = umfPoolGetTag(pool.get(), &tag); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + auto [t, op] = decodeTag(tag); + ASSERT_TRUE(t < NUM_THREADS); + ASSERT_TRUE(op == NUM_OPS_PER_THREAD - 1); +} + +TEST_F(tagTest, SetAndGetInvalidPtr) { + umf_result_t ret = umfPoolSetTag(pool.get(), nullptr, nullptr); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + ret = umfPoolGetTag(pool.get(), nullptr); + ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_F(tagTest, SetAndGetInvalidPool) { + umf_result_t ret = + umfPoolSetTag(nullptr, reinterpret_cast(0x1), nullptr); + ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + void *tag; + ret = umfPoolGetTag(nullptr, &tag); + ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + INSTANTIATE_TEST_SUITE_P( mallocPoolTest, umfPoolTest, ::testing::Values(poolCreateExtParams{&MALLOC_POOL_OPS, nullptr, - &UMF_NULL_PROVIDER_OPS, nullptr, - nullptr}, + &UMF_NULL_PROVIDER_OPS, nullptr}, poolCreateExtParams{umfProxyPoolOps(), nullptr, - &BA_GLOBAL_PROVIDER_OPS, nullptr, - nullptr})); + &BA_GLOBAL_PROVIDER_OPS, nullptr})); INSTANTIATE_TEST_SUITE_P(mallocMultiPoolTest, umfMultiPoolTest, ::testing::Values(poolCreateExtParams{ umfProxyPoolOps(), nullptr, - &BA_GLOBAL_PROVIDER_OPS, nullptr, nullptr})); + &BA_GLOBAL_PROVIDER_OPS, nullptr})); INSTANTIATE_TEST_SUITE_P(umfPoolWithCreateFlagsTest, umfPoolWithCreateFlagsTest, ::testing::Values(0, diff --git a/test/memoryProviderAPI.cpp b/test/memoryProviderAPI.cpp index 866ae6dae3..2dc7261f06 100644 --- a/test/memoryProviderAPI.cpp +++ b/test/memoryProviderAPI.cpp @@ -89,19 +89,6 @@ TEST_F(test, memoryProviderTrace) { ASSERT_EQ(calls.size(), ++call_count); } -TEST_F(test, memoryProviderOpsNullFreeField) { - umf_memory_provider_ops_t provider_ops = UMF_NULL_PROVIDER_OPS; - provider_ops.ext.free = nullptr; - umf_memory_provider_handle_t hProvider; - auto ret = umfMemoryProviderCreate(&provider_ops, nullptr, &hProvider); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - - ret = umfMemoryProviderFree(hProvider, nullptr, 0); - ASSERT_EQ(ret, UMF_RESULT_ERROR_NOT_SUPPORTED); - - umfMemoryProviderDestroy(hProvider); -} - TEST_F(test, memoryProviderOpsNullPurgeLazyField) { umf_memory_provider_ops_t provider_ops = UMF_NULL_PROVIDER_OPS; provider_ops.ext.purge_lazy = nullptr; @@ -204,6 +191,14 @@ TEST_F(test, memoryProviderOpsNullAllocField) { ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); } +TEST_F(test, memoryProviderOpsNullFreeField) { + umf_memory_provider_ops_t provider_ops = UMF_NULL_PROVIDER_OPS; + provider_ops.free = nullptr; + umf_memory_provider_handle_t hProvider; + auto ret = umfMemoryProviderCreate(&provider_ops, nullptr, &hProvider); + ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + TEST_F(test, memoryProviderOpsNullGetLastNativeErrorField) { umf_memory_provider_ops_t provider_ops = UMF_NULL_PROVIDER_OPS; provider_ops.get_last_native_error = nullptr; diff --git a/test/poolFixtures.hpp b/test/poolFixtures.hpp index e5ec85012c..bd97ac1fa6 100644 --- a/test/poolFixtures.hpp +++ b/test/poolFixtures.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -7,7 +7,6 @@ #include "pool.hpp" #include "provider.hpp" -#include "umf/providers/provider_coarse.h" #include "umf/providers/provider_devdax_memory.h" #include "utils/utils_sanitizers.h" @@ -20,13 +19,11 @@ #include "../malloc_compliance_tests.hpp" -using poolCreateExtParams = - std::tuple; +using poolCreateExtParams = std::tuple; umf::pool_unique_handle_t poolCreateExtUnique(poolCreateExtParams params) { - auto [pool_ops, pool_params, provider_ops, provider_params, coarse_params] = - params; + auto [pool_ops, pool_params, provider_ops, provider_params] = params; umf_memory_provider_handle_t upstream_provider = nullptr; umf_memory_provider_handle_t provider = nullptr; @@ -40,22 +37,6 @@ umf::pool_unique_handle_t poolCreateExtUnique(poolCreateExtParams params) { provider = upstream_provider; - if (coarse_params) { - coarse_memory_provider_params_t *coarse_memory_provider_params = - (coarse_memory_provider_params_t *)coarse_params; - coarse_memory_provider_params->upstream_memory_provider = - upstream_provider; - coarse_memory_provider_params->destroy_upstream_memory_provider = true; - - umf_memory_provider_handle_t coarse_provider = nullptr; - ret = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - coarse_params, &coarse_provider); - EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - EXPECT_NE(coarse_provider, nullptr); - - provider = coarse_provider; - } - ret = umfPoolCreate(pool_ops, provider, pool_params, UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &hPool); EXPECT_EQ(ret, UMF_RESULT_SUCCESS); diff --git a/test/pools/disjoint_pool.cpp b/test/pools/disjoint_pool.cpp index 319997c823..c254400db3 100644 --- a/test/pools/disjoint_pool.cpp +++ b/test/pools/disjoint_pool.cpp @@ -29,20 +29,24 @@ disjoint_params_unique_handle_t poolConfig() { res = umfDisjointPoolParamsSetSlabMinSize(config, DEFAULT_DISJOINT_SLAB_MIN_SIZE); if (res != UMF_RESULT_SUCCESS) { + umfDisjointPoolParamsDestroy(config); throw std::runtime_error("Failed to set slab min size"); } res = umfDisjointPoolParamsSetMaxPoolableSize( config, DEFAULT_DISJOINT_MAX_POOLABLE_SIZE); if (res != UMF_RESULT_SUCCESS) { + umfDisjointPoolParamsDestroy(config); throw std::runtime_error("Failed to set max poolable size"); } res = umfDisjointPoolParamsSetCapacity(config, DEFAULT_DISJOINT_CAPACITY); if (res != UMF_RESULT_SUCCESS) { + umfDisjointPoolParamsDestroy(config); throw std::runtime_error("Failed to set capacity"); } res = umfDisjointPoolParamsSetMinBucketSize( config, DEFAULT_DISJOINT_MIN_BUCKET_SIZE); if (res != UMF_RESULT_SUCCESS) { + umfDisjointPoolParamsDestroy(config); throw std::runtime_error("Failed to set min bucket size"); } @@ -244,19 +248,18 @@ INSTANTIATE_TEST_SUITE_P(disjointPoolTests, umfPoolTest, ::testing::Values(poolCreateExtParams{ umfDisjointPoolOps(), (void *)defaultPoolConfig.get(), - &BA_GLOBAL_PROVIDER_OPS, nullptr, nullptr})); + &BA_GLOBAL_PROVIDER_OPS, nullptr})); -INSTANTIATE_TEST_SUITE_P(disjointPoolTests, umfMemTest, - ::testing::Values(std::make_tuple( - poolCreateExtParams{ - umfDisjointPoolOps(), - (void *)defaultPoolConfig.get(), - &MOCK_OUT_OF_MEM_PROVIDER_OPS, - (void *)&DEFAULT_DISJOINT_CAPACITY, nullptr}, - static_cast(DEFAULT_DISJOINT_CAPACITY) / 2))); +INSTANTIATE_TEST_SUITE_P( + disjointPoolTests, umfMemTest, + ::testing::Values(std::make_tuple( + poolCreateExtParams{ + umfDisjointPoolOps(), (void *)defaultPoolConfig.get(), + &MOCK_OUT_OF_MEM_PROVIDER_OPS, (void *)&DEFAULT_DISJOINT_CAPACITY}, + static_cast(DEFAULT_DISJOINT_CAPACITY) / 2))); INSTANTIATE_TEST_SUITE_P(disjointMultiPoolTests, umfMultiPoolTest, ::testing::Values(poolCreateExtParams{ umfDisjointPoolOps(), (void *)defaultPoolConfig.get(), - &BA_GLOBAL_PROVIDER_OPS, nullptr, nullptr})); + &BA_GLOBAL_PROVIDER_OPS, nullptr})); diff --git a/test/pools/jemalloc_coarse_devdax.cpp b/test/pools/jemalloc_coarse_devdax.cpp index 350e053ab7..72906e625b 100644 --- a/test/pools/jemalloc_coarse_devdax.cpp +++ b/test/pools/jemalloc_coarse_devdax.cpp @@ -31,15 +31,13 @@ devdax_params_unique_handle_t create_devdax_params() { &umfDevDaxMemoryProviderParamsDestroy); } -auto coarseParams = umfCoarseMemoryProviderParamsDefault(); auto devdaxParams = create_devdax_params(); static std::vector poolParamsList = - devdaxParams.get() - ? std::vector{poolCreateExtParams{ - umfJemallocPoolOps(), nullptr, umfDevDaxMemoryProviderOps(), - devdaxParams.get(), &coarseParams}} - : std::vector{}; + devdaxParams.get() ? std::vector{poolCreateExtParams{ + umfJemallocPoolOps(), nullptr, + umfDevDaxMemoryProviderOps(), devdaxParams.get()}} + : std::vector{}; INSTANTIATE_TEST_SUITE_P(jemallocCoarseDevDaxTest, umfPoolTest, ::testing::ValuesIn(poolParamsList)); diff --git a/test/pools/jemalloc_coarse_file.cpp b/test/pools/jemalloc_coarse_file.cpp index 74ad36d56e..68a602df64 100644 --- a/test/pools/jemalloc_coarse_file.cpp +++ b/test/pools/jemalloc_coarse_file.cpp @@ -23,11 +23,9 @@ file_params_unique_handle_t get_file_params_default(char *path) { &umfFileMemoryProviderParamsDestroy); } -auto coarseParams = umfCoarseMemoryProviderParamsDefault(); file_params_unique_handle_t fileParams = get_file_params_default(FILE_PATH); INSTANTIATE_TEST_SUITE_P(jemallocCoarseFileTest, umfPoolTest, ::testing::Values(poolCreateExtParams{ umfJemallocPoolOps(), nullptr, - umfFileMemoryProviderOps(), fileParams.get(), - &coarseParams})); + umfFileMemoryProviderOps(), fileParams.get()})); diff --git a/test/pools/jemalloc_pool.cpp b/test/pools/jemalloc_pool.cpp index 4dddbcd32b..86784d919f 100644 --- a/test/pools/jemalloc_pool.cpp +++ b/test/pools/jemalloc_pool.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -29,8 +29,7 @@ auto defaultParams = createOsMemoryProviderParams(); INSTANTIATE_TEST_SUITE_P(jemallocPoolTest, umfPoolTest, ::testing::Values(poolCreateExtParams{ umfJemallocPoolOps(), nullptr, - umfOsMemoryProviderOps(), defaultParams.get(), - nullptr})); + umfOsMemoryProviderOps(), defaultParams.get()})); // this test makes sure that jemalloc does not use // memory provider to allocate metadata (and hence @@ -48,9 +47,8 @@ TEST_F(test, metadataNotAllocatedUsingProvider) { res = umfOsMemoryProviderParamsSetProtection(params, UMF_PROTECTION_NONE); ASSERT_EQ(res, UMF_RESULT_SUCCESS); - auto pool = - poolCreateExtUnique({umfJemallocPoolOps(), nullptr, - umfOsMemoryProviderOps(), params, nullptr}); + auto pool = poolCreateExtUnique( + {umfJemallocPoolOps(), nullptr, umfOsMemoryProviderOps(), params}); res = umfOsMemoryProviderParamsDestroy(params); ASSERT_EQ(res, UMF_RESULT_SUCCESS); @@ -62,122 +60,3 @@ TEST_F(test, metadataNotAllocatedUsingProvider) { [pool = pool.get()](void *ptr) { umfPoolFree(pool, ptr); }); } } - -using jemallocPoolParams = bool; -struct umfJemallocPoolParamsTest - : umf_test::test, - ::testing::WithParamInterface { - - struct validation_params_t { - bool keep_all_memory; - }; - - struct provider_validator : public umf_test::provider_ba_global { - using base_provider = umf_test::provider_ba_global; - - umf_result_t initialize(validation_params_t *params) { - EXPECT_NE(params, nullptr); - expected_params = params; - return UMF_RESULT_SUCCESS; - } - umf_result_t free(void *ptr, size_t size) { - EXPECT_EQ(expected_params->keep_all_memory, false); - return base_provider::free(ptr, size); - } - - validation_params_t *expected_params; - }; - - static constexpr umf_memory_provider_ops_t VALIDATOR_PROVIDER_OPS = - umf::providerMakeCOps(); - - umfJemallocPoolParamsTest() : expected_params{false}, params(nullptr) {} - void SetUp() override { - test::SetUp(); - expected_params.keep_all_memory = this->GetParam(); - umf_result_t ret = umfJemallocPoolParamsCreate(¶ms); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - ret = umfJemallocPoolParamsSetKeepAllMemory( - params, expected_params.keep_all_memory); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - } - - void TearDown() override { - umfJemallocPoolParamsDestroy(params); - test::TearDown(); - } - - umf::pool_unique_handle_t makePool() { - umf_memory_provider_handle_t hProvider = nullptr; - umf_memory_pool_handle_t hPool = nullptr; - - auto ret = umfMemoryProviderCreate(&VALIDATOR_PROVIDER_OPS, - &expected_params, &hProvider); - EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - - ret = umfPoolCreate(umfJemallocPoolOps(), hProvider, params, - UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &hPool); - EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - - return umf::pool_unique_handle_t(hPool, &umfPoolDestroy); - } - - void allocFreeFlow() { - static const size_t ALLOC_SIZE = 128; - static const size_t NUM_ALLOCATIONS = 100; - std::vector ptrs; - - auto pool = makePool(); - ASSERT_NE(pool, nullptr); - - for (size_t i = 0; i < NUM_ALLOCATIONS; ++i) { - auto *ptr = umfPoolMalloc(pool.get(), ALLOC_SIZE); - ASSERT_NE(ptr, nullptr); - ptrs.push_back(ptr); - } - - for (size_t i = 0; i < NUM_ALLOCATIONS; ++i) { - auto ret = umfPoolFree(pool.get(), ptrs[i]); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - } - - // Now pool can call free during pool destruction - expected_params.keep_all_memory = false; - } - - validation_params_t expected_params; - umf_jemalloc_pool_params_handle_t params; -}; - -TEST_P(umfJemallocPoolParamsTest, allocFree) { allocFreeFlow(); } - -TEST_P(umfJemallocPoolParamsTest, updateParams) { - expected_params.keep_all_memory = !expected_params.keep_all_memory; - umf_result_t ret = umfJemallocPoolParamsSetKeepAllMemory( - params, expected_params.keep_all_memory); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - - allocFreeFlow(); -} - -TEST_P(umfJemallocPoolParamsTest, invalidParams) { - umf_result_t ret = umfJemallocPoolParamsCreate(nullptr); - ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - ret = umfJemallocPoolParamsSetKeepAllMemory(nullptr, true); - ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - ret = umfJemallocPoolParamsSetKeepAllMemory(nullptr, false); - ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - ret = umfJemallocPoolParamsDestroy(nullptr); - ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); -} - -GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(umfJemallocPoolParamsTest); - -/* TODO: enable this test after the issue #903 is fixed. -(https://github.com/oneapi-src/unified-memory-framework/issues/903) -INSTANTIATE_TEST_SUITE_P(jemallocPoolTest, umfJemallocPoolParamsTest, - testing::Values(false, true)); -*/ diff --git a/test/pools/pool_base_alloc.cpp b/test/pools/pool_base_alloc.cpp index 7c9a3701a7..752d9f01e2 100644 --- a/test/pools/pool_base_alloc.cpp +++ b/test/pools/pool_base_alloc.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -48,4 +48,4 @@ umf_memory_pool_ops_t BA_POOL_OPS = umf::poolMakeCOps(); INSTANTIATE_TEST_SUITE_P(baPool, umfPoolTest, ::testing::Values(poolCreateExtParams{ &BA_POOL_OPS, nullptr, - &umf_test::BASE_PROVIDER_OPS, nullptr, nullptr})); + &umf_test::BASE_PROVIDER_OPS, nullptr})); diff --git a/test/pools/pool_coarse.hpp b/test/pools/pool_coarse.hpp index 7baa612f1e..b1efb4fee9 100644 --- a/test/pools/pool_coarse.hpp +++ b/test/pools/pool_coarse.hpp @@ -5,8 +5,6 @@ #ifndef UMF_TEST_POOL_COARSE_HPP #define UMF_TEST_POOL_COARSE_HPP 1 -#include "umf/providers/provider_coarse.h" - #include "pool.hpp" #include "poolFixtures.hpp" diff --git a/test/pools/scalable_coarse_devdax.cpp b/test/pools/scalable_coarse_devdax.cpp index 1bf77c61ce..970f45ef92 100644 --- a/test/pools/scalable_coarse_devdax.cpp +++ b/test/pools/scalable_coarse_devdax.cpp @@ -31,15 +31,13 @@ devdax_params_unique_handle_t create_devdax_params() { &umfDevDaxMemoryProviderParamsDestroy); } -auto coarseParams = umfCoarseMemoryProviderParamsDefault(); auto devdaxParams = create_devdax_params(); static std::vector poolParamsList = - devdaxParams.get() - ? std::vector{poolCreateExtParams{ - umfScalablePoolOps(), nullptr, umfDevDaxMemoryProviderOps(), - devdaxParams.get(), &coarseParams}} - : std::vector{}; + devdaxParams.get() ? std::vector{poolCreateExtParams{ + umfScalablePoolOps(), nullptr, + umfDevDaxMemoryProviderOps(), devdaxParams.get()}} + : std::vector{}; INSTANTIATE_TEST_SUITE_P(scalableCoarseDevDaxTest, umfPoolTest, ::testing::ValuesIn(poolParamsList)); diff --git a/test/pools/scalable_coarse_file.cpp b/test/pools/scalable_coarse_file.cpp index b45c112be2..30134f5eba 100644 --- a/test/pools/scalable_coarse_file.cpp +++ b/test/pools/scalable_coarse_file.cpp @@ -23,11 +23,9 @@ file_params_unique_handle_t get_file_params_default(char *path) { &umfFileMemoryProviderParamsDestroy); } -auto coarseParams = umfCoarseMemoryProviderParamsDefault(); file_params_unique_handle_t fileParams = get_file_params_default(FILE_PATH); INSTANTIATE_TEST_SUITE_P(scalableCoarseFileTest, umfPoolTest, ::testing::Values(poolCreateExtParams{ umfScalablePoolOps(), nullptr, - umfFileMemoryProviderOps(), fileParams.get(), - &coarseParams})); + umfFileMemoryProviderOps(), fileParams.get()})); diff --git a/test/pools/scalable_pool.cpp b/test/pools/scalable_pool.cpp index 3edacd965c..ce55923d9a 100644 --- a/test/pools/scalable_pool.cpp +++ b/test/pools/scalable_pool.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -27,8 +27,7 @@ auto defaultParams = createOsMemoryProviderParams(); INSTANTIATE_TEST_SUITE_P(scalablePoolTest, umfPoolTest, ::testing::Values(poolCreateExtParams{ umfScalablePoolOps(), nullptr, - umfOsMemoryProviderOps(), defaultParams.get(), - nullptr})); + umfOsMemoryProviderOps(), defaultParams.get()})); using scalablePoolParams = std::tuple; struct umfScalablePoolParamsTest diff --git a/test/provider_coarse.cpp b/test/provider_coarse.cpp deleted file mode 100644 index c2de4c06a9..0000000000 --- a/test/provider_coarse.cpp +++ /dev/null @@ -1,668 +0,0 @@ -/* - * Copyright (C) 2023-2024 Intel Corporation - * - * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -*/ - -#include - -#include "provider.hpp" - -#include - -using umf_test::KB; -using umf_test::MB; -using umf_test::test; - -#define GetStats umfCoarseMemoryProviderGetStats - -#define UPSTREAM_NAME "umf_ba_global" -#define BASE_NAME "coarse" -#define COARSE_NAME BASE_NAME " (" UPSTREAM_NAME ")" - -umf_memory_provider_ops_t UMF_MALLOC_MEMORY_PROVIDER_OPS = - umf::providerMakeCOps(); - -struct CoarseWithMemoryStrategyTest - : umf_test::test, - ::testing::WithParamInterface { - void SetUp() override { - test::SetUp(); - allocation_strategy = this->GetParam(); - } - - coarse_memory_provider_strategy_t allocation_strategy; -}; - -INSTANTIATE_TEST_SUITE_P( - CoarseWithMemoryStrategyTest, CoarseWithMemoryStrategyTest, - ::testing::Values(UMF_COARSE_MEMORY_STRATEGY_FASTEST, - UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE, - UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE)); - -TEST_F(test, coarseProvider_name_upstream) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - const size_t init_buffer_size = 20 * MB; - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.destroy_upstream_memory_provider = true; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = nullptr; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - size_t minPageSize = 0; - umf_result = umfMemoryProviderGetMinPageSize(coarse_memory_provider, - nullptr, &minPageSize); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_UNKNOWN); - ASSERT_EQ(minPageSize, 0); - - size_t pageSize = 0; - umf_result = umfMemoryProviderGetRecommendedPageSize( - coarse_memory_provider, minPageSize, &pageSize); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_UNKNOWN); - ASSERT_EQ(pageSize, minPageSize); - - ASSERT_EQ( - strcmp(umfMemoryProviderGetName(coarse_memory_provider), COARSE_NAME), - 0); - - umfMemoryProviderDestroy(coarse_memory_provider); - // malloc_memory_provider has already been destroyed - // by umfMemoryProviderDestroy(coarse_memory_provider), because: - // coarse_memory_provider_params.destroy_upstream_memory_provider = true; -} - -TEST_F(test, coarseProvider_name_no_upstream) { - umf_result_t umf_result; - - const size_t init_buffer_size = 20 * MB; - - // preallocate some memory and initialize the vector with zeros - std::vector buffer(init_buffer_size, 0); - void *buf = (void *)buffer.data(); - ASSERT_NE(buf, nullptr); - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.upstream_memory_provider = nullptr; - coarse_memory_provider_params.immediate_init_from_upstream = false; - coarse_memory_provider_params.init_buffer = buf; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - size_t minPageSize = 0; - umf_result = umfMemoryProviderGetMinPageSize(coarse_memory_provider, - nullptr, &minPageSize); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_GT(minPageSize, 0); - - size_t pageSize = 0; - umf_result = umfMemoryProviderGetRecommendedPageSize( - coarse_memory_provider, minPageSize, &pageSize); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_GE(pageSize, minPageSize); - - ASSERT_EQ( - strcmp(umfMemoryProviderGetName(coarse_memory_provider), BASE_NAME), 0); - - umfMemoryProviderDestroy(coarse_memory_provider); -} - -// negative tests - -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_null_stats) { - ASSERT_EQ(GetStats(nullptr).alloc_size, 0); - ASSERT_EQ(GetStats(nullptr).used_size, 0); - ASSERT_EQ(GetStats(nullptr).num_upstream_blocks, 0); - ASSERT_EQ(GetStats(nullptr).num_all_blocks, 0); - ASSERT_EQ(GetStats(nullptr).num_free_blocks, 0); -} - -// wrong NULL parameters -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_NULL_params) { - umf_result_t umf_result; - - umf_memory_provider_handle_t coarse_memory_provider = nullptr; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), nullptr, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - ASSERT_EQ(coarse_memory_provider, nullptr); -} - -// wrong parameters: given no upstream_memory_provider -// nor init_buffer while exactly one of them must be set -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_wrong_params_0) { - umf_result_t umf_result; - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = nullptr; - coarse_memory_provider_params.immediate_init_from_upstream = false; - coarse_memory_provider_params.init_buffer = nullptr; - coarse_memory_provider_params.init_buffer_size = 0; - - umf_memory_provider_handle_t coarse_memory_provider = nullptr; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - ASSERT_EQ(coarse_memory_provider, nullptr); -} - -// wrong parameters: given both an upstream_memory_provider -// and an init_buffer while only one of them is allowed -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_wrong_params_1) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - const size_t init_buffer_size = 20 * MB; - - // preallocate some memory and initialize the vector with zeros - std::vector buffer(init_buffer_size, 0); - void *buf = (void *)buffer.data(); - ASSERT_NE(buf, nullptr); - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = buf; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider = nullptr; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - ASSERT_EQ(coarse_memory_provider, nullptr); - - umfMemoryProviderDestroy(malloc_memory_provider); -} - -// wrong parameters: init_buffer_size must not equal 0 when immediate_init_from_upstream is true -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_wrong_params_2) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = nullptr; - coarse_memory_provider_params.init_buffer_size = 0; - - umf_memory_provider_handle_t coarse_memory_provider = nullptr; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - ASSERT_EQ(coarse_memory_provider, nullptr); - - umfMemoryProviderDestroy(malloc_memory_provider); -} - -// wrong parameters: init_buffer_size must not equal 0 when init_buffer is not NULL -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_wrong_params_3) { - umf_result_t umf_result; - - const size_t init_buffer_size = 20 * MB; - - // preallocate some memory and initialize the vector with zeros - std::vector buffer(init_buffer_size, 0); - void *buf = (void *)buffer.data(); - ASSERT_NE(buf, nullptr); - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = nullptr; - coarse_memory_provider_params.immediate_init_from_upstream = false; - coarse_memory_provider_params.init_buffer = buf; - coarse_memory_provider_params.init_buffer_size = 0; - - umf_memory_provider_handle_t coarse_memory_provider = nullptr; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - ASSERT_EQ(coarse_memory_provider, nullptr); -} - -// wrong parameters: init_buffer_size must equal 0 when init_buffer is NULL and immediate_init_from_upstream is false -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_wrong_params_4) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.immediate_init_from_upstream = false; - coarse_memory_provider_params.init_buffer = NULL; - coarse_memory_provider_params.init_buffer_size = 20 * MB; - - umf_memory_provider_handle_t coarse_memory_provider = nullptr; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - ASSERT_EQ(coarse_memory_provider, nullptr); - - umfMemoryProviderDestroy(malloc_memory_provider); -} - -// wrong parameters: destroy_upstream_memory_provider is true, but an upstream provider is not provided -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_wrong_params_5) { - umf_result_t umf_result; - - const size_t init_buffer_size = 20 * MB; - - // preallocate some memory and initialize the vector with zeros - std::vector buffer(init_buffer_size, 0); - void *buf = (void *)buffer.data(); - ASSERT_NE(buf, nullptr); - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = nullptr; - coarse_memory_provider_params.destroy_upstream_memory_provider = true; - coarse_memory_provider_params.immediate_init_from_upstream = false; - coarse_memory_provider_params.init_buffer = buf; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider = nullptr; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - ASSERT_EQ(coarse_memory_provider, nullptr); -} - -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_split_merge) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - const size_t init_buffer_size = 20 * MB; - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = NULL; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - umf_memory_provider_handle_t cp = coarse_memory_provider; - char *ptr = nullptr; - - ASSERT_EQ(GetStats(cp).used_size, 0 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 1); - - /* test umfMemoryProviderAllocationSplit */ - umf_result = umfMemoryProviderAlloc(cp, 2 * MB, 0, (void **)&ptr); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(ptr, nullptr); - ASSERT_EQ(GetStats(cp).used_size, 2 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 2); - - umf_result = umfMemoryProviderAllocationSplit(cp, ptr, 2 * MB, 1 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 2 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 3); - - umf_result = umfMemoryProviderFree(cp, (ptr + 1 * MB), 1 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 1 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 2); - - umf_result = umfMemoryProviderFree(cp, ptr, 1 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 0); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 1); - - /* test umfMemoryProviderAllocationMerge */ - umf_result = umfMemoryProviderAlloc(cp, 2 * MB, 0, (void **)&ptr); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(ptr, nullptr); - ASSERT_EQ(GetStats(cp).used_size, 2 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 2); - - umf_result = umfMemoryProviderAllocationSplit(cp, ptr, 2 * MB, 1 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 2 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 3); - - umf_result = - umfMemoryProviderAllocationMerge(cp, ptr, (ptr + 1 * MB), 2 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 2 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 2); - - umf_result = umfMemoryProviderFree(cp, ptr, 2 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 0); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 1); - - umfMemoryProviderDestroy(coarse_memory_provider); - umfMemoryProviderDestroy(malloc_memory_provider); -} - -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_split_merge_negative) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - const size_t init_buffer_size = 20 * MB; - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = NULL; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - umf_memory_provider_handle_t cp = coarse_memory_provider; - char *ptr = nullptr; - - ASSERT_EQ(GetStats(cp).used_size, 0 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 1); - - /* test umfMemoryProviderAllocationSplit */ - umf_result = umfMemoryProviderAlloc(cp, 6 * MB, 0, (void **)&ptr); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(ptr, nullptr); - ASSERT_EQ(GetStats(cp).used_size, 6 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 2); - - // firstSize >= totalSize - umf_result = umfMemoryProviderAllocationSplit(cp, ptr, 6 * MB, 6 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // firstSize == 0 - umf_result = umfMemoryProviderAllocationSplit(cp, ptr, 6 * MB, 0); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // wrong totalSize - umf_result = umfMemoryProviderAllocationSplit(cp, ptr, 5 * MB, 1 * KB); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - /* test umfMemoryProviderAllocationMerge */ - // split (6 * MB) block into (1 * MB) + (5 * MB) - umf_result = umfMemoryProviderAllocationSplit(cp, ptr, 6 * MB, 1 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 6 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 3); - - // split (5 * MB) block into (2 * MB) + (3 * MB) - umf_result = - umfMemoryProviderAllocationSplit(cp, (ptr + 1 * MB), 5 * MB, 2 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 6 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 4); - - // now we have 3 blocks: (1 * MB) + (2 * MB) + (3 * MB) - - // highPtr <= lowPtr - umf_result = - umfMemoryProviderAllocationMerge(cp, (ptr + 1 * MB), ptr, 2 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // highPtr - lowPtr >= totalSize - umf_result = - umfMemoryProviderAllocationMerge(cp, ptr, (ptr + 1 * MB), 1 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // low_block->size + high_block->size != totalSize - umf_result = - umfMemoryProviderAllocationMerge(cp, ptr, (ptr + 1 * MB), 5 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // not adjacent blocks - umf_result = - umfMemoryProviderAllocationMerge(cp, ptr, (ptr + 3 * MB), 4 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - umf_result = umfMemoryProviderFree(cp, ptr, 1 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 5 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 4); - - umf_result = umfMemoryProviderFree(cp, (ptr + 1 * MB), 2 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 3 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 3); - - umf_result = umfMemoryProviderFree(cp, (ptr + 3 * MB), 3 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 0); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 1); - - umfMemoryProviderDestroy(coarse_memory_provider); - umfMemoryProviderDestroy(malloc_memory_provider); -} - -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_purge_no_upstream) { - umf_result_t umf_result; - - const size_t init_buffer_size = 20 * MB; - - // preallocate some memory and initialize the vector with zeros - std::vector buffer(init_buffer_size, 0); - void *buf = (void *)buffer.data(); - ASSERT_NE(buf, nullptr); - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = nullptr; - coarse_memory_provider_params.immediate_init_from_upstream = false; - coarse_memory_provider_params.init_buffer = buf; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider = nullptr; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - // umfMemoryProviderPurgeLazy - // provider == NULL - umf_result = umfMemoryProviderPurgeLazy(nullptr, (void *)0x01, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // ptr == NULL - umf_result = umfMemoryProviderPurgeLazy(coarse_memory_provider, nullptr, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // no upstream_memory_provider - umf_result = - umfMemoryProviderPurgeLazy(coarse_memory_provider, (void *)0x01, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); - - // umfMemoryProviderPurgeForce - // provider == NULL - umf_result = umfMemoryProviderPurgeForce(nullptr, (void *)0x01, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // ptr == NULL - umf_result = - umfMemoryProviderPurgeForce(coarse_memory_provider, nullptr, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // no upstream_memory_provider - umf_result = - umfMemoryProviderPurgeForce(coarse_memory_provider, (void *)0x01, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); - - umfMemoryProviderDestroy(coarse_memory_provider); -} - -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_purge_with_upstream) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - const size_t init_buffer_size = 20 * MB; - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = NULL; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - // umfMemoryProviderPurgeLazy - // provider == NULL - umf_result = umfMemoryProviderPurgeLazy(nullptr, (void *)0x01, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // ptr == NULL - umf_result = umfMemoryProviderPurgeLazy(coarse_memory_provider, nullptr, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // malloc_memory_provider returns UMF_RESULT_ERROR_UNKNOWN - umf_result = - umfMemoryProviderPurgeLazy(coarse_memory_provider, (void *)0x01, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_UNKNOWN); - - // umfMemoryProviderPurgeForce - // provider == NULL - umf_result = umfMemoryProviderPurgeForce(nullptr, (void *)0x01, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // ptr == NULL - umf_result = - umfMemoryProviderPurgeForce(coarse_memory_provider, nullptr, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // malloc_memory_provider returns UMF_RESULT_ERROR_UNKNOWN - umf_result = - umfMemoryProviderPurgeForce(coarse_memory_provider, (void *)0x01, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_UNKNOWN); - - umfMemoryProviderDestroy(coarse_memory_provider); - umfMemoryProviderDestroy(malloc_memory_provider); -} diff --git a/test/provider_devdax_memory.cpp b/test/provider_devdax_memory.cpp index 0fd0705da4..7765dd08da 100644 --- a/test/provider_devdax_memory.cpp +++ b/test/provider_devdax_memory.cpp @@ -100,7 +100,7 @@ static void test_alloc_free_success(umf_memory_provider_handle_t provider, } umf_result = umfMemoryProviderFree(provider, ptr, size); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } static void verify_last_native_error(umf_memory_provider_handle_t provider, @@ -162,9 +162,10 @@ TEST_F(test, test_if_mapped_with_MAP_SYNC) { bool flag_found = is_mapped_with_MAP_SYNC(path, buf, size); umf_result = umfMemoryProviderFree(hProvider, buf, size); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); umfMemoryProviderDestroy(hProvider); + umfDevDaxMemoryProviderParamsDestroy(params); // fail test if the "sf" flag was not found ASSERT_EQ(flag_found, true); @@ -233,34 +234,40 @@ TEST_P(umfProviderTest, purge_force) { test_alloc_free_success(provider.get(), page_size, 0, PURGE_FORCE); } +TEST_P(umfProviderTest, purge_force_unalligned_alloc) { + void *ptr; + auto ret = umfMemoryProviderAlloc(provider.get(), page_plus_64, 0, &ptr); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + test_alloc_free_success(provider.get(), page_size, 0, PURGE_FORCE); + umfMemoryProviderFree(provider.get(), ptr, page_plus_64); +} // negative tests using test_alloc_failure TEST_P(umfProviderTest, alloc_page64_align_page_minus_1_WRONG_ALIGNMENT_1) { test_alloc_failure(provider.get(), page_plus_64, page_size - 1, - UMF_RESULT_ERROR_INVALID_ARGUMENT, 0); + UMF_RESULT_ERROR_INVALID_ALIGNMENT, 0); } TEST_P(umfProviderTest, alloc_page64_align_one_half_pages_WRONG_ALIGNMENT_2) { test_alloc_failure(provider.get(), page_plus_64, page_size + (page_size / 2), - UMF_RESULT_ERROR_INVALID_ARGUMENT, 0); + UMF_RESULT_ERROR_INVALID_ALIGNMENT, 0); } TEST_P(umfProviderTest, alloc_page64_WRONG_ALIGNMENT_3_pages) { test_alloc_failure(provider.get(), page_plus_64, 3 * page_size, - UMF_RESULT_ERROR_INVALID_ARGUMENT, 0); + UMF_RESULT_ERROR_INVALID_ALIGNMENT, 0); } TEST_P(umfProviderTest, alloc_3_pages_WRONG_ALIGNMENT_3_pages) { test_alloc_failure(provider.get(), 3 * page_size, 3 * page_size, - UMF_RESULT_ERROR_INVALID_ARGUMENT, 0); + UMF_RESULT_ERROR_INVALID_ALIGNMENT, 0); } TEST_P(umfProviderTest, alloc_WRONG_SIZE) { size_t size = (size_t)(-1) & ~(page_size - 1); test_alloc_failure(provider.get(), size, 0, - UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC, - UMF_DEVDAX_RESULT_ERROR_ALLOC_FAILED); + UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY, 0); } // other positive tests @@ -295,12 +302,12 @@ TEST_P(umfProviderTest, get_name) { TEST_P(umfProviderTest, free_size_0_ptr_not_null) { umf_result_t umf_result = umfMemoryProviderFree(provider.get(), INVALID_PTR, 0); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); } TEST_P(umfProviderTest, free_NULL) { umf_result_t umf_result = umfMemoryProviderFree(provider.get(), nullptr, 0); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } // other negative tests @@ -308,7 +315,7 @@ TEST_P(umfProviderTest, free_NULL) { TEST_P(umfProviderTest, free_INVALID_POINTER_SIZE_GT_0) { umf_result_t umf_result = umfMemoryProviderFree(provider.get(), INVALID_PTR, page_plus_64); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); } TEST_P(umfProviderTest, purge_lazy_INVALID_POINTER) { diff --git a/test/provider_devdax_memory_ipc.cpp b/test/provider_devdax_memory_ipc.cpp index 3941f66e90..ed4f1a5f8d 100644 --- a/test/provider_devdax_memory_ipc.cpp +++ b/test/provider_devdax_memory_ipc.cpp @@ -53,14 +53,14 @@ static std::vector getIpcProxyPoolTestParamsList(void) { ipcProxyPoolTestParamsList = { {umfProxyPoolOps(), nullptr, umfDevDaxMemoryProviderOps(), - defaultDevDaxParams.get(), &hostAccessor, true}, + defaultDevDaxParams.get(), &hostAccessor}, #ifdef UMF_POOL_JEMALLOC_ENABLED {umfJemallocPoolOps(), nullptr, umfDevDaxMemoryProviderOps(), - defaultDevDaxParams.get(), &hostAccessor, false}, + defaultDevDaxParams.get(), &hostAccessor}, #endif #ifdef UMF_POOL_SCALABLE_ENABLED {umfScalablePoolOps(), nullptr, umfDevDaxMemoryProviderOps(), - defaultDevDaxParams.get(), &hostAccessor, false}, + defaultDevDaxParams.get(), &hostAccessor}, #endif }; diff --git a/test/provider_file_memory.cpp b/test/provider_file_memory.cpp index d3124aa11f..cfa37be31a 100644 --- a/test/provider_file_memory.cpp +++ b/test/provider_file_memory.cpp @@ -98,7 +98,7 @@ static void test_alloc_free_success(umf_memory_provider_handle_t provider, } umf_result = umfMemoryProviderFree(provider, ptr, size); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } static void verify_last_native_error(umf_memory_provider_handle_t provider, @@ -159,9 +159,10 @@ TEST_F(test, test_if_mapped_with_MAP_SYNC) { bool flag_found = is_mapped_with_MAP_SYNC(path, buf, size); umf_result = umfMemoryProviderFree(hProvider, buf, size); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); umfMemoryProviderDestroy(hProvider); + umfFileMemoryProviderParamsDestroy(params); // fail test if the "sf" flag was not found ASSERT_EQ(flag_found, true); @@ -244,10 +245,10 @@ TEST_P(FileProviderParamsDefault, two_allocations) { memset(ptr2, 0x22, size); umf_result = umfMemoryProviderFree(provider.get(), ptr1, size); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); umf_result = umfMemoryProviderFree(provider.get(), ptr2, size); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } TEST_P(FileProviderParamsDefault, alloc_page64_align_0) { @@ -366,12 +367,12 @@ TEST_P(FileProviderParamsDefault, get_name) { TEST_P(FileProviderParamsDefault, free_size_0_ptr_not_null) { umf_result_t umf_result = umfMemoryProviderFree(provider.get(), INVALID_PTR, 0); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); } TEST_P(FileProviderParamsDefault, free_NULL) { umf_result_t umf_result = umfMemoryProviderFree(provider.get(), nullptr, 0); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } // other negative tests @@ -449,7 +450,7 @@ TEST_F(test, set_null_path) { TEST_P(FileProviderParamsDefault, free_INVALID_POINTER_SIZE_GT_0) { umf_result_t umf_result = umfMemoryProviderFree(provider.get(), INVALID_PTR, page_plus_64); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); } TEST_P(FileProviderParamsDefault, purge_lazy_INVALID_POINTER) { @@ -512,7 +513,7 @@ TEST_P(FileProviderParamsShared, IPC_base_success_test) { ASSERT_EQ(ret, 0); umf_result = umfMemoryProviderFree(provider.get(), ptr, size); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } TEST_P(FileProviderParamsShared, IPC_file_not_exist) { @@ -552,5 +553,5 @@ TEST_P(FileProviderParamsShared, IPC_file_not_exist) { ASSERT_EQ(new_ptr, nullptr); umf_result = umfMemoryProviderFree(provider.get(), ptr, size); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } diff --git a/test/provider_file_memory_ipc.cpp b/test/provider_file_memory_ipc.cpp index ee7ab6c8ff..70c1acd8f5 100644 --- a/test/provider_file_memory_ipc.cpp +++ b/test/provider_file_memory_ipc.cpp @@ -73,14 +73,14 @@ HostMemoryAccessor hostAccessor; static std::vector ipcManyPoolsTestParamsList = { // TODO: enable it when sizes of allocations in ipcFixtures.hpp are fixed // {umfProxyPoolOps(), nullptr, umfFileMemoryProviderOps(), -// file_params_shared.get(), &hostAccessor, true}, +// file_params_shared.get(), &hostAccessor}, #ifdef UMF_POOL_JEMALLOC_ENABLED {umfJemallocPoolOps(), nullptr, umfFileMemoryProviderOps(), - file_params_shared.get(), &hostAccessor, false}, + file_params_shared.get(), &hostAccessor}, #endif #ifdef UMF_POOL_SCALABLE_ENABLED {umfScalablePoolOps(), nullptr, umfFileMemoryProviderOps(), - file_params_shared.get(), &hostAccessor, false}, + file_params_shared.get(), &hostAccessor}, #endif }; @@ -96,14 +96,14 @@ static std::vector getIpcFsDaxTestParamsList(void) { ipcFsDaxTestParamsList = { // TODO: enable it when sizes of allocations in ipcFixtures.hpp are fixed // {umfProxyPoolOps(), nullptr, umfFileMemoryProviderOps(), -// file_params_fsdax.get(), &hostAccessor, true}, +// file_params_fsdax.get(), &hostAccessor}, #ifdef UMF_POOL_JEMALLOC_ENABLED {umfJemallocPoolOps(), nullptr, umfFileMemoryProviderOps(), - file_params_fsdax.get(), &hostAccessor, false}, + file_params_fsdax.get(), &hostAccessor}, #endif #ifdef UMF_POOL_SCALABLE_ENABLED {umfScalablePoolOps(), nullptr, umfFileMemoryProviderOps(), - file_params_fsdax.get(), &hostAccessor, false}, + file_params_fsdax.get(), &hostAccessor}, #endif }; diff --git a/test/provider_fixed_memory.cpp b/test/provider_fixed_memory.cpp new file mode 100644 index 0000000000..7f976a1f5d --- /dev/null +++ b/test/provider_fixed_memory.cpp @@ -0,0 +1,393 @@ +// Copyright (C) 2024 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "base.hpp" + +#include "cpp_helpers.hpp" +#include "test_helpers.h" +#ifndef _WIN32 +#include "test_helpers_linux.h" +#endif + +#include +#include + +using umf_test::test; + +#define INVALID_PTR ((void *)0x01) + +typedef enum purge_t { + PURGE_NONE = 0, + PURGE_LAZY = 1, + PURGE_FORCE = 2, +} purge_t; + +static const char *Native_error_str[] = { + "success", // UMF_FIXED_RESULT_SUCCESS + "force purging failed", // UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED +}; + +// Test helpers + +static int compare_native_error_str(const char *message, int error) { + const char *error_str = Native_error_str[error - UMF_FIXED_RESULT_SUCCESS]; + size_t len = strlen(error_str); + return strncmp(message, error_str, len); +} + +using providerCreateExtParams = std::tuple; + +static void providerCreateExt(providerCreateExtParams params, + umf::provider_unique_handle_t *handle) { + umf_memory_provider_handle_t hProvider = nullptr; + auto [provider_ops, provider_params] = params; + + auto ret = + umfMemoryProviderCreate(provider_ops, provider_params, &hProvider); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(hProvider, nullptr); + + *handle = + umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); +} + +struct FixedProviderTest + : umf_test::test, + ::testing::WithParamInterface { + void SetUp() override { + test::SetUp(); + + // Allocate a memory buffer to use with the fixed memory provider + memory_size = utils_get_page_size() * 10; // Allocate 10 pages + memory_buffer = malloc(memory_size); + ASSERT_NE(memory_buffer, nullptr); + + // Create provider parameters + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result_t res = umfFixedMemoryProviderParamsCreate( + ¶ms, memory_buffer, memory_size); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + providerCreateExt(std::make_tuple(umfFixedMemoryProviderOps(), params), + &provider); + + umfFixedMemoryProviderParamsDestroy(params); + umf_result_t umf_result = + umfMemoryProviderGetMinPageSize(provider.get(), NULL, &page_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + page_plus_64 = page_size + 64; + } + + void TearDown() override { + if (memory_buffer) { + free(memory_buffer); + memory_buffer = nullptr; + } + test::TearDown(); + } + + void test_alloc_free_success(size_t size, size_t alignment, purge_t purge) { + void *ptr = nullptr; + auto provider = this->provider.get(); + + umf_result_t umf_result = + umfMemoryProviderAlloc(provider, size, alignment, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + + memset(ptr, 0xFF, size); + + if (purge == PURGE_LAZY) { + umf_result = umfMemoryProviderPurgeLazy(provider, ptr, size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + } else if (purge == PURGE_FORCE) { + umf_result = umfMemoryProviderPurgeForce(provider, ptr, size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + + umf_result = umfMemoryProviderFree(provider, ptr, size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + + void verify_last_native_error(int32_t err) { + const char *message; + int32_t error; + auto provider = this->provider.get(); + umfMemoryProviderGetLastNativeError(provider, &message, &error); + ASSERT_EQ(error, err); + ASSERT_EQ(compare_native_error_str(message, error), 0); + } + + void test_alloc_failure(size_t size, size_t alignment, umf_result_t result, + int32_t err) { + void *ptr = nullptr; + auto provider = this->provider.get(); + + umf_result_t umf_result = + umfMemoryProviderAlloc(provider, size, alignment, &ptr); + ASSERT_EQ(umf_result, result); + ASSERT_EQ(ptr, nullptr); + + if (umf_result == UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC) { + verify_last_native_error(err); + } + } + + umf::provider_unique_handle_t provider; + size_t page_size; + size_t page_plus_64; + void *memory_buffer = nullptr; + size_t memory_size = 0; +}; + +// TESTS + +// Positive tests using test_alloc_free_success + +INSTANTIATE_TEST_SUITE_P(fixedProviderTest, FixedProviderTest, + ::testing::Values(providerCreateExtParams{ + umfFixedMemoryProviderOps(), nullptr})); + +TEST_P(FixedProviderTest, create_destroy) { + // Creation and destruction are handled in SetUp and TearDown +} + +TEST_F(test, create_no_params) { + umf_memory_provider_handle_t provider = nullptr; + auto result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), nullptr, + &provider); + ASSERT_EQ(result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(provider, nullptr); +} + +TEST_P(FixedProviderTest, two_allocations) { + umf_result_t umf_result; + void *ptr1 = nullptr; + void *ptr2 = nullptr; + size_t size = page_plus_64; + size_t alignment = page_size; + + umf_result = umfMemoryProviderAlloc(provider.get(), size, alignment, &ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr1, nullptr); + + umf_result = umfMemoryProviderAlloc(provider.get(), size, alignment, &ptr2); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr2, nullptr); + + ASSERT_NE(ptr1, ptr2); + if ((uintptr_t)ptr1 > (uintptr_t)ptr2) { + ASSERT_GT((uintptr_t)ptr1 - (uintptr_t)ptr2, size); + } else { + ASSERT_GT((uintptr_t)ptr2 - (uintptr_t)ptr1, size); + } + + memset(ptr1, 0x11, size); + memset(ptr2, 0x22, size); + + umf_result = umfMemoryProviderFree(provider.get(), ptr1, size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_result = umfMemoryProviderFree(provider.get(), ptr2, size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(FixedProviderTest, alloc_page64_align_0) { + test_alloc_free_success(page_plus_64, 0, PURGE_NONE); +} + +TEST_P(FixedProviderTest, alloc_page64_align_page_div_2) { + test_alloc_free_success(page_plus_64, page_size / 2, PURGE_NONE); +} + +TEST_P(FixedProviderTest, purge_lazy) { + test_alloc_free_success(page_size, 0, PURGE_LAZY); +} + +TEST_P(FixedProviderTest, purge_force) { + test_alloc_free_success(page_size, 0, PURGE_FORCE); +} + +// Negative tests using test_alloc_failure + +TEST_P(FixedProviderTest, alloc_WRONG_SIZE) { + test_alloc_failure((size_t)-1, 0, UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY, 0); +} + +TEST_P(FixedProviderTest, alloc_page64_WRONG_ALIGNMENT_3_pages) { + test_alloc_failure(page_plus_64, 3 * page_size, + UMF_RESULT_ERROR_INVALID_ALIGNMENT, 0); +} + +TEST_P(FixedProviderTest, alloc_3pages_WRONG_ALIGNMENT_3pages) { + test_alloc_failure(3 * page_size, 3 * page_size, + UMF_RESULT_ERROR_INVALID_ALIGNMENT, 0); +} + +TEST_P(FixedProviderTest, alloc_page64_align_page_plus_1_WRONG_ALIGNMENT_1) { + test_alloc_failure(page_plus_64, page_size + 1, + UMF_RESULT_ERROR_INVALID_ALIGNMENT, 0); +} + +TEST_P(FixedProviderTest, alloc_page64_align_one_half_pages_WRONG_ALIGNMENT_2) { + test_alloc_failure(page_plus_64, page_size + (page_size / 2), + UMF_RESULT_ERROR_INVALID_ALIGNMENT, 0); +} + +// Other positive tests + +TEST_P(FixedProviderTest, get_min_page_size) { + size_t min_page_size; + umf_result_t umf_result = umfMemoryProviderGetMinPageSize( + provider.get(), nullptr, &min_page_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_LE(min_page_size, page_size); +} + +TEST_P(FixedProviderTest, get_recommended_page_size) { + size_t min_page_size; + umf_result_t umf_result = umfMemoryProviderGetMinPageSize( + provider.get(), nullptr, &min_page_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_LE(min_page_size, page_size); + + size_t recommended_page_size; + umf_result = umfMemoryProviderGetRecommendedPageSize( + provider.get(), 0, &recommended_page_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_GE(recommended_page_size, min_page_size); +} + +TEST_P(FixedProviderTest, get_name) { + const char *name = umfMemoryProviderGetName(provider.get()); + ASSERT_STREQ(name, "FIXED"); +} + +TEST_P(FixedProviderTest, free_size_0_ptr_not_null) { + umf_result_t umf_result = + umfMemoryProviderFree(provider.get(), INVALID_PTR, 0); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_P(FixedProviderTest, free_NULL) { + umf_result_t umf_result = umfMemoryProviderFree(provider.get(), nullptr, 0); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +// Other negative tests + +TEST_P(FixedProviderTest, free_INVALID_POINTER_SIZE_GT_0) { + umf_result_t umf_result = + umfMemoryProviderFree(provider.get(), INVALID_PTR, page_plus_64); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_P(FixedProviderTest, purge_lazy_INVALID_POINTER) { + umf_result_t umf_result = + umfMemoryProviderPurgeLazy(provider.get(), INVALID_PTR, 1); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); +} + +TEST_P(FixedProviderTest, purge_force_INVALID_POINTER) { + umf_result_t umf_result = + umfMemoryProviderPurgeForce(provider.get(), INVALID_PTR, 1); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC); + + verify_last_native_error(UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED); +} + +// Params tests + +TEST_F(test, params_null_handle) { + constexpr size_t memory_size = 100; + char memory_buffer[memory_size]; + umf_result_t umf_result = + umfFixedMemoryProviderParamsCreate(nullptr, memory_buffer, memory_size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + umf_result = umfFixedMemoryProviderParamsDestroy(nullptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_F(test, create_with_null_ptr) { + constexpr size_t memory_size = 100; + umf_fixed_memory_provider_params_handle_t wrong_params = nullptr; + umf_result_t umf_result = + umfFixedMemoryProviderParamsCreate(&wrong_params, nullptr, memory_size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(wrong_params, nullptr); +} + +TEST_F(test, create_with_zero_size) { + constexpr size_t memory_size = 100; + char memory_buffer[memory_size]; + umf_fixed_memory_provider_params_handle_t wrong_params = nullptr; + umf_result_t umf_result = + umfFixedMemoryProviderParamsCreate(&wrong_params, memory_buffer, 0); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(wrong_params, nullptr); +} + +TEST_P(FixedProviderTest, alloc_size_exceeds_buffer) { + size_t size = memory_size + page_size; + test_alloc_failure(size, 0, UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY, 0); +} + +TEST_P(FixedProviderTest, merge) { + umf_result_t umf_result; + void *ptr1 = nullptr; + void *ptr2 = nullptr; + size_t size = page_size; + size_t alignment = page_size; + + umf_result = umfMemoryProviderAlloc(provider.get(), size, alignment, &ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr1, nullptr); + + umf_result = umfMemoryProviderAlloc(provider.get(), size, alignment, &ptr2); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr2, nullptr); + + ASSERT_EQ((uintptr_t)ptr2 - (uintptr_t)ptr1, size); + + memset(ptr1, 0x11, size); + memset(ptr2, 0x22, size); + + size_t merged_size = size * 2; + umf_result = umfMemoryProviderAllocationMerge(provider.get(), ptr1, ptr2, + merged_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_result = umfMemoryProviderFree(provider.get(), ptr1, merged_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(FixedProviderTest, split) { + umf_result_t umf_result; + void *ptr1 = nullptr; + void *ptr2 = nullptr; + size_t size = page_size; + size_t alignment = page_size; + + umf_result = + umfMemoryProviderAlloc(provider.get(), size * 2, alignment, &ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr1, nullptr); + + umf_result = + umfMemoryProviderAllocationSplit(provider.get(), ptr1, size * 2, size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ptr2 = (void *)((uintptr_t)ptr1 + size); + memset(ptr1, 0x11, size); + + umf_result = umfMemoryProviderFree(provider.get(), ptr1, size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + memset(ptr2, 0x22, size); + umf_result = umfMemoryProviderFree(provider.get(), ptr2, size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} diff --git a/test/provider_os_memory.cpp b/test/provider_os_memory.cpp index 57bce46d24..9544a6feda 100644 --- a/test/provider_os_memory.cpp +++ b/test/provider_os_memory.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -441,18 +441,22 @@ disjoint_params_unique_handle_t disjointPoolParams() { } res = umfDisjointPoolParamsSetSlabMinSize(params, 4096); if (res != UMF_RESULT_SUCCESS) { + umfDisjointPoolParamsDestroy(params); throw std::runtime_error("Failed to set slab min size"); } res = umfDisjointPoolParamsSetMaxPoolableSize(params, 4096); if (res != UMF_RESULT_SUCCESS) { + umfDisjointPoolParamsDestroy(params); throw std::runtime_error("Failed to set max poolable size"); } res = umfDisjointPoolParamsSetCapacity(params, 4); if (res != UMF_RESULT_SUCCESS) { + umfDisjointPoolParamsDestroy(params); throw std::runtime_error("Failed to set capacity"); } res = umfDisjointPoolParamsSetMinBucketSize(params, 64); if (res != UMF_RESULT_SUCCESS) { + umfDisjointPoolParamsDestroy(params); throw std::runtime_error("Failed to set min bucket size"); } @@ -465,11 +469,11 @@ disjoint_params_unique_handle_t disjointParams = disjointPoolParams(); static std::vector ipcTestParamsList = { #if (defined UMF_POOL_DISJOINT_ENABLED) {umfDisjointPoolOps(), disjointParams.get(), umfOsMemoryProviderOps(), - os_params.get(), &hostAccessor, false}, + os_params.get(), &hostAccessor}, #endif #ifdef UMF_POOL_JEMALLOC_ENABLED {umfJemallocPoolOps(), nullptr, umfOsMemoryProviderOps(), os_params.get(), - &hostAccessor, false}, + &hostAccessor}, #endif }; diff --git a/test/provider_os_memory_multiple_numa_nodes.cpp b/test/provider_os_memory_multiple_numa_nodes.cpp index e493a427ca..cfc58f2f06 100644 --- a/test/provider_os_memory_multiple_numa_nodes.cpp +++ b/test/provider_os_memory_multiple_numa_nodes.cpp @@ -674,17 +674,17 @@ TEST_P(testNumaSplit, checkModeSplit) { auto [required_numa_nodes, pages, in, out] = param; umf_result_t umf_result; - umf_os_memory_provider_params_handle_t os_memory_provider_params = nullptr; - - umf_result = umfOsMemoryProviderParamsCreate(&os_memory_provider_params); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - std::vector numa_nodes = get_available_numa_nodes(); if (numa_nodes.size() < required_numa_nodes) { GTEST_SKIP_("Not enough numa nodes"); } + umf_os_memory_provider_params_handle_t os_memory_provider_params = nullptr; + + umf_result = umfOsMemoryProviderParamsCreate(&os_memory_provider_params); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(out.size(), pages) << "Wrong test input - out array size doesn't match page count"; diff --git a/test/providers/ipc_level_zero_prov.sh b/test/providers/ipc_level_zero_prov.sh index d6bcef4f3a..4d29677257 100755 --- a/test/providers/ipc_level_zero_prov.sh +++ b/test/providers/ipc_level_zero_prov.sh @@ -1,5 +1,5 @@ # -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2024-2025 Intel Corporation # # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -12,21 +12,6 @@ set -e # port should be a number from the range <1024, 65535> PORT=$(( 1024 + ( $$ % ( 65535 - 1024 )))) -# The ipc_level_zero_prov test requires using pidfd_getfd(2) -# to obtain a duplicate of another process's file descriptor. -# Permission to duplicate another process's file descriptor -# is governed by a ptrace access mode PTRACE_MODE_ATTACH_REALCREDS check (see ptrace(2)) -# that can be changed using the /proc/sys/kernel/yama/ptrace_scope interface. -PTRACE_SCOPE_FILE="/proc/sys/kernel/yama/ptrace_scope" -VAL=0 -if [ -f $PTRACE_SCOPE_FILE ]; then - PTRACE_SCOPE_VAL=$(cat $PTRACE_SCOPE_FILE) - if [ $PTRACE_SCOPE_VAL -ne $VAL ]; then - echo "SKIP: ptrace_scope is not set to 0 (classic ptrace permissions) - skipping the test" - exit 125 # skip code defined in CMakeLists.txt - fi -fi - UMF_LOG_VAL="level:debug;flush:debug;output:stderr;pid:yes" echo "Starting ipc_level_zero_prov CONSUMER on port $PORT ..." diff --git a/test/providers/provider_level_zero.cpp b/test/providers/provider_level_zero.cpp index d0584777be..78b5e4847a 100644 --- a/test/providers/provider_level_zero.cpp +++ b/test/providers/provider_level_zero.cpp @@ -454,9 +454,9 @@ INSTANTIATE_TEST_SUITE_P( #ifdef _WIN32 GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(umfIpcTest); #else -INSTANTIATE_TEST_SUITE_P( - umfLevelZeroProviderTestSuite, umfIpcTest, - ::testing::Values(ipcTestParams{ - umfProxyPoolOps(), nullptr, umfLevelZeroMemoryProviderOps(), - l0Params_device_memory.get(), &l0Accessor, false})); +INSTANTIATE_TEST_SUITE_P(umfLevelZeroProviderTestSuite, umfIpcTest, + ::testing::Values(ipcTestParams{ + umfProxyPoolOps(), nullptr, + umfLevelZeroMemoryProviderOps(), + l0Params_device_memory.get(), &l0Accessor})); #endif diff --git a/test/providers/provider_level_zero_not_impl.cpp b/test/providers/provider_level_zero_not_impl.cpp index bea1acbe79..c55c236fe7 100644 --- a/test/providers/provider_level_zero_not_impl.cpp +++ b/test/providers/provider_level_zero_not_impl.cpp @@ -31,6 +31,10 @@ TEST_F(test, level_zero_provider_not_implemented) { hDevices, 1); ASSERT_EQ(result, UMF_RESULT_ERROR_NOT_SUPPORTED); + result = umfLevelZeroMemoryProviderParamsSetFreePolicy( + hParams, UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_DEFAULT); + ASSERT_EQ(result, UMF_RESULT_ERROR_NOT_SUPPORTED); + umf_memory_provider_ops_t *ops = umfLevelZeroMemoryProviderOps(); ASSERT_EQ(ops, nullptr); } diff --git a/test/supp/drd-umf_test-jemalloc_coarse_devdax.supp b/test/supp/drd-umf_test-jemalloc_coarse_devdax.supp index fd071432b6..bc4f2295f4 100644 --- a/test/supp/drd-umf_test-jemalloc_coarse_devdax.supp +++ b/test/supp/drd-umf_test-jemalloc_coarse_devdax.supp @@ -1,27 +1,8 @@ { - False-positive ConflictingAccess in libjemalloc.so + False-positive ConflictingAccess in jemalloc drd:ConflictingAccess - obj:*/libjemalloc.so* ... - fun:mallocx - ... -} - -{ - False-positive ConflictingAccess in libjemalloc.so - drd:ConflictingAccess - obj:*/libjemalloc.so* - ... - fun:op_free - ... -} - -{ - False-positive ConflictingAccess in libjemalloc.so - drd:ConflictingAccess - obj:*/libjemalloc.so* - ... - fun:__nptl_deallocate_tsd + fun:je_* ... } diff --git a/test/supp/drd-umf_test-jemalloc_coarse_file.supp b/test/supp/drd-umf_test-jemalloc_coarse_file.supp index fd071432b6..bc4f2295f4 100644 --- a/test/supp/drd-umf_test-jemalloc_coarse_file.supp +++ b/test/supp/drd-umf_test-jemalloc_coarse_file.supp @@ -1,27 +1,8 @@ { - False-positive ConflictingAccess in libjemalloc.so + False-positive ConflictingAccess in jemalloc drd:ConflictingAccess - obj:*/libjemalloc.so* ... - fun:mallocx - ... -} - -{ - False-positive ConflictingAccess in libjemalloc.so - drd:ConflictingAccess - obj:*/libjemalloc.so* - ... - fun:op_free - ... -} - -{ - False-positive ConflictingAccess in libjemalloc.so - drd:ConflictingAccess - obj:*/libjemalloc.so* - ... - fun:__nptl_deallocate_tsd + fun:je_* ... } diff --git a/test/supp/drd-umf_test-jemalloc_pool.supp b/test/supp/drd-umf_test-jemalloc_pool.supp index 965ef38844..cb6179f875 100644 --- a/test/supp/drd-umf_test-jemalloc_pool.supp +++ b/test/supp/drd-umf_test-jemalloc_pool.supp @@ -1,6 +1,7 @@ { - Conflicting Access in libjemalloc.so - internal issue of libjemalloc + False-positive ConflictingAccess in jemalloc drd:ConflictingAccess - obj:*libjemalloc.so* + ... + fun:je_* ... } diff --git a/test/supp/helgrind-umf_test-jemalloc_coarse_devdax.supp b/test/supp/helgrind-umf_test-jemalloc_coarse_devdax.supp index 18774f387c..ac8969c5ae 100644 --- a/test/supp/helgrind-umf_test-jemalloc_coarse_devdax.supp +++ b/test/supp/helgrind-umf_test-jemalloc_coarse_devdax.supp @@ -1,27 +1,8 @@ { - False-positive Race in libjemalloc.so + False-positive Race in jemalloc Helgrind:Race - obj:*/libjemalloc.so* ... - fun:mallocx - ... -} - -{ - False-positive Race in libjemalloc.so - Helgrind:Race - obj:*/libjemalloc.so* - ... - fun:op_free - ... -} - -{ - False-positive Race in libjemalloc.so - Helgrind:Race - obj:*/libjemalloc.so* - ... - fun:__nptl_deallocate_tsd + fun:je_* ... } diff --git a/test/supp/helgrind-umf_test-jemalloc_coarse_file.supp b/test/supp/helgrind-umf_test-jemalloc_coarse_file.supp index 18774f387c..ac8969c5ae 100644 --- a/test/supp/helgrind-umf_test-jemalloc_coarse_file.supp +++ b/test/supp/helgrind-umf_test-jemalloc_coarse_file.supp @@ -1,27 +1,8 @@ { - False-positive Race in libjemalloc.so + False-positive Race in jemalloc Helgrind:Race - obj:*/libjemalloc.so* ... - fun:mallocx - ... -} - -{ - False-positive Race in libjemalloc.so - Helgrind:Race - obj:*/libjemalloc.so* - ... - fun:op_free - ... -} - -{ - False-positive Race in libjemalloc.so - Helgrind:Race - obj:*/libjemalloc.so* - ... - fun:__nptl_deallocate_tsd + fun:je_* ... } diff --git a/test/supp/helgrind-umf_test-jemalloc_pool.supp b/test/supp/helgrind-umf_test-jemalloc_pool.supp index 8068b023dc..98d748feac 100644 --- a/test/supp/helgrind-umf_test-jemalloc_pool.supp +++ b/test/supp/helgrind-umf_test-jemalloc_pool.supp @@ -1,6 +1,7 @@ { - Race in libjemalloc.so - internal issue of libjemalloc + False-positive Race in jemalloc Helgrind:Race - obj:*libjemalloc.so* + ... + fun:je_* ... } diff --git a/test/supp/memcheck-umf_test-jemalloc_coarse_devdax.supp b/test/supp/memcheck-umf_test-jemalloc_coarse_devdax.supp new file mode 100644 index 0000000000..f719032779 --- /dev/null +++ b/test/supp/memcheck-umf_test-jemalloc_coarse_devdax.supp @@ -0,0 +1,7 @@ +{ + False-positive invalid write of size 8 + Memcheck:Addr8 + ... + fun:je_* + ... +} diff --git a/test/supp/memcheck-umf_test-jemalloc_coarse_file.supp b/test/supp/memcheck-umf_test-jemalloc_coarse_file.supp new file mode 100644 index 0000000000..f719032779 --- /dev/null +++ b/test/supp/memcheck-umf_test-jemalloc_coarse_file.supp @@ -0,0 +1,7 @@ +{ + False-positive invalid write of size 8 + Memcheck:Addr8 + ... + fun:je_* + ... +} diff --git a/test/test_installation.py b/test/test_installation.py index 49a382969c..b5dd676dc1 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -283,11 +283,6 @@ def parse_arguments(self) -> argparse.Namespace: action="store_true", help="Add this argument if the UMF was built with Disjoint Pool enabled", ) - self.parser.add_argument( - "--jemalloc-pool", - action="store_true", - help="Add this argument if the UMF was built with Jemalloc Pool enabled", - ) self.parser.add_argument( "--umf-version", action="store", @@ -306,8 +301,6 @@ def run(self) -> None: pools = [] if self.args.disjoint_pool: pools.append("disjoint_pool") - if self.args.jemalloc_pool: - pools.append("jemalloc_pool") umf_version = Version(self.args.umf_version) diff --git a/test/test_valgrind.sh b/test/test_valgrind.sh index 9f84cf0d32..954a3a56bb 100755 --- a/test/test_valgrind.sh +++ b/test/test_valgrind.sh @@ -8,11 +8,16 @@ set -e WORKSPACE=$1 BUILD_DIR=$2 TOOL=$3 +TESTS=$4 function print_usage() { - echo "$(basename $0) - run all UMF tests under a valgrind tool (memcheck, drd or helgrind)" - echo "This script looks for './test/umf_test-*' test executables in the UMF build directory." - echo "Usage: $(basename $0) " + echo "$(basename $0) - run UMF tests and examples under a valgrind tool (memcheck, drd or helgrind)" + echo "Usage: $(basename $0) [tests_examples]" + echo "Where:" + echo + echo "tests_examples - (optional) list of tests or examples to be run (paths relative to the build directory)." + echo " If it is empty, all tests (./test/umf_test-*) and examples (./examples/umf_example_*)" + echo " found in will be run." } if ! valgrind --version > /dev/null; then @@ -58,33 +63,44 @@ esac WORKSPACE=$(realpath $WORKSPACE) BUILD_DIR=$(realpath $BUILD_DIR) -cd ${BUILD_DIR}/test/ +cd ${BUILD_DIR} mkdir -p cpuid echo "Gathering data for hwloc so it can be run under valgrind:" -hwloc-gather-cpuid ./cpuid +hwloc-gather-cpuid ./cpuid >/dev/null echo echo "Working directory: $(pwd)" echo "Running: \"valgrind $OPTION\" for the following tests:" ANY_TEST_FAILED=0 -rm -f umf_test-*.log umf_test-*.err +PATH_TESTS="./test/umf_test-*" +PATH_EXAMPLES="./examples/umf_example_*" + +rm -f ${PATH_TESTS}.log ${PATH_TESTS}.err ${PATH_EXAMPLES}.log ${PATH_EXAMPLES}.err -for test in $(ls -1 umf_test-*); do +[ "$TESTS" = "" ] && TESTS=$(ls -1 ${PATH_TESTS} ${PATH_EXAMPLES}) + +for test in $TESTS; do + if [ ! -f $test ]; then + echo + echo "error: the $test (${BUILD_DIR}/$test) file does not exist" + exit 1 + fi [ ! -x $test ] && continue echo "$test - starting ..." echo -n "$test " LOG=${test}.log ERR=${test}.err - SUP="${WORKSPACE}/test/supp/${TOOL}-${test}.supp" + NAME=$(basename $test) + SUP="${WORKSPACE}/test/supp/${TOOL}-${NAME}.supp" OPT_SUP="" - [ -f ${SUP} ] && OPT_SUP="--suppressions=${SUP}" && echo -n "(${TOOL}-${test}.supp) " + [ -f ${SUP} ] && OPT_SUP="--suppressions=${SUP}" && echo -n "($(basename ${SUP})) " # skip tests incompatible with valgrind FILTER="" case $test in - umf_test-disjointPool) + ./test/umf_test-disjointPool) if [ "$TOOL" = "helgrind" ]; then # skip because of the assert in helgrind: # Helgrind: hg_main.c:308 (lockN_acquire_reader): Assertion 'lk->kind == LK_rdwr' failed. @@ -92,53 +108,61 @@ for test in $(ls -1 umf_test-*); do continue; fi ;; - umf_test-ipc_os_prov_*) + ./test/umf_test-ipc_os_prov_*) echo "- SKIPPED" continue; # skip testing helper binaries used by the ipc_os_prov_* tests ;; - umf_test-ipc_devdax_prov_*) + ./test/umf_test-ipc_devdax_prov_*) echo "- SKIPPED" continue; # skip testing helper binaries used by the ipc_devdax_prov_* tests ;; - umf_test-ipc_file_prov_*) + ./test/umf_test-ipc_file_prov_*) echo "- SKIPPED" continue; # skip testing helper binaries used by the ipc_file_prov_* tests ;; - umf_test-memspace_host_all) + ./test/umf_test-memspace_host_all) FILTER='--gtest_filter="-*allocsSpreadAcrossAllNumaNodes"' ;; - umf_test-provider_os_memory) + ./test/umf_test-provider_os_memory) FILTER='--gtest_filter="-osProviderTest/umfIpcTest*"' ;; - umf_test-provider_os_memory_config) + ./test/umf_test-provider_os_memory_config) FILTER='--gtest_filter="-*protection_flag_none:*protection_flag_read:*providerConfigTestNumaMode*"' ;; - umf_test-memspace_highest_capacity) + ./test/umf_test-memspace_highest_capacity) FILTER='--gtest_filter="-*highestCapacityVerify*"' ;; - umf_test-provider_os_memory_multiple_numa_nodes) + ./test/umf_test-provider_os_memory_multiple_numa_nodes) FILTER='--gtest_filter="-testNuma.checkModeInterleave*:testNumaNodesAllocations/testNumaOnEachNode.checkNumaNodesAllocations*:testNumaNodesAllocations/testNumaOnEachNode.checkModePreferred*:testNumaNodesAllocations/testNumaOnEachNode.checkModeInterleaveSingleNode*:testNumaNodesAllocationsAllCpus/testNumaOnEachCpu.checkModePreferredEmptyNodeset*:testNumaNodesAllocationsAllCpus/testNumaOnEachCpu.checkModeLocal*"' ;; - umf_test-memspace_highest_bandwidth) + ./test/umf_test-memspace_highest_bandwidth) FILTER='--gtest_filter="-*allocLocalMt*"' ;; - umf_test-memspace_lowest_latency) + ./test/umf_test-memspace_lowest_latency) FILTER='--gtest_filter="-*allocLocalMt*"' ;; - umf_test-memoryPool) + ./test/umf_test-memoryPool) FILTER='--gtest_filter="-*allocMaxSize*"' ;; + ./examples/umf_example_ipc_ipcapi_*) + echo "- SKIPPED" + continue; # skip testing helper binaries used by the umf_example_ipc_ipcapi_* examples + ;; esac [ "$FILTER" != "" ] && echo -n "($FILTER) " LAST_TEST_FAILED=0 - - if ! HWLOC_CPUID_PATH=./cpuid valgrind $OPTION $OPT_SUP --gen-suppressions=all ./$test $FILTER >$LOG 2>&1; then + set +e + HWLOC_CPUID_PATH=./cpuid valgrind $OPTION $OPT_SUP --gen-suppressions=all $test $FILTER >$LOG 2>&1 + RET=$? + set -e + # 125 is the return code when the test is skipped + if [ $RET -ne 0 -a $RET -ne 125 ]; then LAST_TEST_FAILED=1 ANY_TEST_FAILED=1 - echo "(valgrind FAILED) " - echo "Command: HWLOC_CPUID_PATH=./cpuid valgrind $OPTION $OPT_SUP --gen-suppressions=all ./$test $FILTER >$LOG 2>&1" + echo "(valgrind FAILED RV=$RET) " + echo "Command: HWLOC_CPUID_PATH=./cpuid valgrind $OPTION $OPT_SUP --gen-suppressions=all $test $FILTER >$LOG 2>&1" echo "Output:" cat $LOG echo "=====================" @@ -147,7 +171,7 @@ for test in $(ls -1 umf_test-*); do # grep for "ERROR SUMMARY" with errors (there can be many lines with "ERROR SUMMARY") grep -e "ERROR SUMMARY:" $LOG | grep -v -e "ERROR SUMMARY: 0 errors from 0 contexts" > $ERR || true if [ $LAST_TEST_FAILED -eq 0 -a $(cat $ERR | wc -l) -eq 0 ]; then - echo "- OK" + [ $RET -eq 0 ] && echo "- OK" || echo "- SKIPPED" rm -f $LOG $ERR else echo "- FAILED!" @@ -164,7 +188,7 @@ echo echo "======================================================================" echo -for log in $(ls -1 umf_test-*.log); do +for log in $(ls -1 ${PATH_TESTS}.log ${PATH_EXAMPLES}.log); do echo ">>>>>>> LOG $log" cat $log echo diff --git a/third_party/requirements.txt b/third_party/requirements.txt index 6a8be6e46a..1255dcb929 100644 --- a/third_party/requirements.txt +++ b/third_party/requirements.txt @@ -6,7 +6,7 @@ black==24.3.0 # Tests packaging==24.2 # Generating HTML documentation -pygments==2.18.0 +pygments==2.19.1 sphinxcontrib_applehelp==2.0.0 sphinxcontrib_devhelp==2.0.0 sphinxcontrib_htmlhelp==2.1.0 @@ -15,3 +15,6 @@ sphinxcontrib_qthelp==2.0.0 breathe==4.35.0 sphinx==8.1.3 sphinx_book_theme==1.1.3 +# Spelling check in documentation +pyenchant==3.2.2 +sphinxcontrib-spelling==8.0.0