diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml deleted file mode 100644 index acb64034b..000000000 --- a/.github/workflows/bandit.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Bandit static analysis (for Python code) -name: Bandit -on: - push: - branches-ignore: - - 'dependabot/**' - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - bandit: - name: Bandit - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ (matrix.os == 'ubuntu-latest' && github.repository_owner == 'oneapi-src') && 'intel-ubuntu-22.04' || matrix.os }} - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Install Bandit - run: python3 -m pip install bandit - - # Run Bandit recursively, but omit _deps directory (with 3rd party code) - - name: Run Bandit - run: python3 -m bandit -r . -x '/_deps/' diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index b449eb23e..000000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,93 +0,0 @@ -# CodeQL static analysis -name: CodeQL - -# Due to lower score on Scorecard we're running this separately from -# "PR/push" workflow. For some reason permissions weren't properly set -# or recognized (by Scorecard). If Scorecard changes its behavior we can -# go back to use 'workflow_call' trigger. -on: - push: - branches-ignore: - - 'dependabot/**' - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -env: - BUILD_DIR : "${{github.workspace}}/build" - INSTL_DIR : "${{github.workspace}}/../install-dir" - -jobs: - analyze: - name: Analyze - permissions: - security-events: write - env: - VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows" - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] - include: - - os: ubuntu-latest - # Windows doesn't recognize 'CMAKE_BUILD_TYPE', it uses '--config' param in build command - extra_build_option: '-DCMAKE_BUILD_TYPE=Release' - - os: windows-latest - runs-on: ${{matrix.os}} - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Initialize CodeQL - uses: github/codeql-action/init@b7bf0a3ed3ecfa44160715d7c442788f65f0f923 # v3.23.2 - with: - languages: cpp - - - name: Initialize vcpkg - if: ${{ matrix.os == 'windows-latest' }} - uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5 - with: - vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289 - vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg - vcpkgJsonGlob: '**/vcpkg.json' - - - name: Install dependencies - if: ${{ matrix.os == 'windows-latest' }} - run: vcpkg install - shell: pwsh # Specifies PowerShell as the shell for running the script. - - - name: Install apt packages - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt-get update - sudo apt-get install -y cmake clang libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev - - - name: Install pip packages - run: python3 -m pip install -r third_party/requirements.txt - - - name: Configure CMake - run: > - cmake - -B ${{env.BUILD_DIR}} - ${{matrix.extra_build_option}} - -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" - -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" - -DUMF_FORMAT_CODE_STYLE=OFF - -DUMF_DEVELOPER_MODE=ON - -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON - -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON - -DUMF_TESTS_FAIL_ON_SKIP=ON - - - name: Build - run: cmake --build ${{env.BUILD_DIR}} --config Release -j - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@b7bf0a3ed3ecfa44160715d7c442788f65f0f923 # v3.23.2 diff --git a/.github/workflows/devdax.yml b/.github/workflows/devdax.yml new file mode 100644 index 000000000..b0be853e9 --- /dev/null +++ b/.github/workflows/devdax.yml @@ -0,0 +1,66 @@ +# This workflow builds and tests the devdax memory provider. +# It requires a DAX device (e.g. /dev/dax0.0) configured in the OS. +# This DAX device should be specified using DEVDAX_PATH and DEVDAX_SIZE +# CI environment variables. + +name: DevDAX + +on: [push, pull_request] + +permissions: + contents: read + +env: + DEVDAX_PATH : "/dev/dax0.0" + DEVDAX_SIZE : 1054867456 + BUILD_DIR : "${{github.workspace}}/build" + INSTL_DIR : "${{github.workspace}}/../install-dir" + +jobs: + devdax: + name: Build + # run only on upstream; forks may not have a DAX device + if: github.repository == 'oneapi-src/unified-memory-framework' + strategy: + matrix: + os: ['Ubuntu'] + build_type: [Debug] + shared_library: ['ON', 'OFF'] + compiler: [{c: gcc, cxx: g++}] + number_of_processors: ['$(nproc)'] + + runs-on: ["DSS-DEVDAX", "DSS-${{matrix.os}}"] + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + + - name: Configure build for Ubuntu + run: > + cmake + -B ${{env.BUILD_DIR}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" + -DCMAKE_BUILD_TYPE=${{matrix.build_type}} + -DCMAKE_C_COMPILER=${{matrix.compiler.c}} + -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}} + -DUMF_BUILD_SHARED_LIBRARY=${{matrix.shared_library}} + -DUMF_BUILD_BENCHMARKS=OFF + -DUMF_BUILD_TESTS=ON + -DUMF_BUILD_GPU_TESTS=OFF + -DUMF_BUILD_GPU_EXAMPLES=OFF + -DUMF_FORMAT_CODE_STYLE=OFF + -DUMF_DEVELOPER_MODE=ON + -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON + -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON + -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF + -DUMF_TESTS_FAIL_ON_SKIP=ON + -DUMF_TESTS_DEVDAX_PATH=${{env.DEVDAX_PATH}} + -DUMF_TESTS_DEVDAX_SIZE=${{env.DEVDAX_SIZE}} + + - name: Build UMF + run: cmake --build ${{env.BUILD_DIR}} --config ${{matrix.build_type}} -j ${{matrix.number_of_processors}} + + - name: Run only devdax tests + working-directory: ${{env.BUILD_DIR}} + run: ctest -C ${{matrix.build_type}} -R devdax -V diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml deleted file mode 100644 index c42d6e098..000000000 --- a/.github/workflows/pr_push.yml +++ /dev/null @@ -1,108 +0,0 @@ -# Checks required for a PR to merge. This workflow mostly call other workflows. -name: PR/push - -on: - push: - branches-ignore: - - 'dependabot/**' - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - CodeStyle: - name: Coding style - runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }} - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Install apt packages - run: | - sudo apt-get update - sudo apt-get install -y black cmake clang-format-15 cmake-format libhwloc-dev - - - name: Configure CMake - run: > - cmake - -B ${{github.workspace}}/build - -DUMF_FORMAT_CODE_STYLE=ON - -DUMF_BUILD_TESTS=OFF - -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF - -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=OFF - - - name: Check C/C++ formatting - run: cmake --build build --target clang-format-check - - - name: Check CMake formatting - run: | - cmake --build build --target cmake-format-apply - git diff --exit-code - - - name: Check Python formatting - run: cmake --build build --target black-format-check - - DocsBuild: - name: Build docs - runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }} - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Install doxygen - run: | - sudo apt-get update - sudo apt-get install -y doxygen - - - name: Install pip requirements - run: python3 -m pip install -r third_party/requirements.txt - - - name: Setup PATH for python - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Build the documentation - working-directory: scripts - run: python3 generate_docs.py - - Spellcheck: - uses: ./.github/workflows/spellcheck.yml - FastBuild: - name: Fast builds - needs: [Spellcheck, CodeStyle] - uses: ./.github/workflows/fast.yml - Build: - name: Basic builds - needs: [FastBuild] - uses: ./.github/workflows/basic.yml - Sanitizers: - needs: [FastBuild] - uses: ./.github/workflows/sanitizers.yml - Qemu: - needs: [FastBuild] - uses: ./.github/workflows/qemu.yml - Benchmarks: - needs: [Build] - uses: ./.github/workflows/benchmarks.yml - ProxyLib: - needs: [Build] - uses: ./.github/workflows/proxy_lib.yml - GPU: - needs: [Build] - uses: ./.github/workflows/gpu.yml - Valgrind: - needs: [Build] - uses: ./.github/workflows/valgrind.yml - MultiNuma: - needs: [Build] - uses: ./.github/workflows/multi_numa.yml diff --git a/.github/workflows/qemu.yml b/.github/workflows/qemu.yml deleted file mode 100644 index f8916c7de..000000000 --- a/.github/workflows/qemu.yml +++ /dev/null @@ -1,112 +0,0 @@ -# Builds project on qemu with custom hmat settings -name: Qemu - -on: workflow_call - -env: - CI_BRANCH: "${{ github.head_ref || github.ref_name }}" - -permissions: - contents: read - -jobs: - qemu-build: - name: Qemu - runs-on: ubuntu-22.04 - - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Enable KVM - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - name: Install qemu - run: | - sudo apt update && sudo apt install -y qemu-system genisoimage qemu-utils - - name: Install libvirt and script dependencies - run: | - sudo apt-get install -y libvirt-clients libvirt-daemon-system libvirt-daemon virtinst bridge-utils - pip install -r scripts/qemu/requirements.txt - sudo usermod -a -G kvm,libvirt $USER - - name: Run ssh-keygen - run: ssh-keygen -b 4096 -N '' -f ~/.ssh/id_rsa - - name: Generate iso with user info - run: | - pub_key=$(cat ~/.ssh/id_rsa.pub) - - cat > user-data << EOF - #cloud-config - - # Add a 'cxltest' user to the system with a password - users: - - default - - name: cxltest - gecos: CXL Test User - primary_group: wheel - groups: users - sudo: ALL=(ALL) NOPASSWD:ALL - lock_passwd: false - ssh-authorized-keys: - - $pub_key - shell: /usr/bin/bash - - # Set local logins - chpasswd: - list: | - root:password - cxltest:password - expire: False - EOF - - cat > meta-data << EOF - instance-id: cxl-test - local-hostname: cxl-test - EOF - - sudo -Sk genisoimage -output ubuntu-cloud-init.iso -volid cidata -joliet -rock ./user-data ./meta-data - - name: Download ubuntu image - run: wget https://cloud-images.ubuntu.com/releases/lunar/release/ubuntu-23.04-server-cloudimg-amd64.img - - name: Resize image - run: qemu-img resize ./ubuntu-23.04-server-cloudimg-amd64.img +4G - - name: Build - run: | - scripts/qemu/start_qemu.sh scripts/qemu/configs/default.xml - - if [ ${{ github.event_name }} = 'pull_request' ]; then - CI_REPO="${{ github.event.pull_request.head.repo.full_name }}" - else - CI_REPO="$GITHUB_REPOSITORY" - fi - - scp -P 2222 ${{github.workspace}}/scripts/qemu/run-build.sh cxltest@127.0.0.1:/home/cxltest - scp -P 2222 ${{github.workspace}}/scripts/qemu/run-tests.sh cxltest@127.0.0.1:/home/cxltest - ssh cxltest@127.0.0.1 -p 2222 -t "bash /home/cxltest/run-build.sh https://github.com/$CI_REPO ${{env.CI_BRANCH}}" - - ssh cxltest@127.0.0.1 -p 2222 -t "sudo shutdown -h now" - - - name: Run tests - run: | - for config_file in scripts/qemu/configs/*.xml; do - config_name=$(basename $config_file .xml) - - echo testing $config_name - while ps -aux | grep qemu-system-x86_64 | grep -q -v grep; do - echo "Waiting for QEMU to shut down..." - sleep 5 - done - scripts/qemu/start_qemu.sh $config_file - - if [ ${{ github.event_name }} = 'pull_request' ]; then - CI_REPO="${{ github.event.pull_request.head.repo.full_name }}" - else - CI_REPO="$GITHUB_REPOSITORY" - fi - - ssh cxltest@127.0.0.1 -p 2222 -t "bash /home/cxltest/run-tests.sh" - ssh cxltest@127.0.0.1 -p 2222 -t "sudo shutdown -h now" - done diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml deleted file mode 100644 index 21a76d0cd..000000000 --- a/.github/workflows/trivy.yml +++ /dev/null @@ -1,57 +0,0 @@ -# Runs linter for Docker files -name: Trivy - -# Due to lower score on Scorecard we're running this separately from -# "PR/push" workflow. For some reason permissions weren't properly set -# or recognized (by Scorecard). If Scorecard changes its behavior we can -# use 'workflow_call' trigger. -on: - push: - branches-ignore: - - 'dependabot/**' - pull_request: - paths: - - '.github/docker/*Dockerfile' - - '.github/workflows/trivy.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - trivy: - name: Trivy - runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }} - permissions: - security-events: write - - steps: - - name: Clone the git repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Run Trivy - uses: aquasecurity/trivy-action@84384bd6e777ef152729993b8145ea352e9dd3ef # v0.17.0 - with: - scan-type: 'config' - hide-progress: false - format: 'sarif' - output: 'trivy-results.sarif' - exit-code: 1 # Fail if issue found - # file with suppressions: .trivyignore (in root dir) - - - name: Print report and trivyignore file - run: | - echo "### Trivy ignore content:" - cat .trivyignore - echo "### Trivy report:" - cat trivy-results.sarif - - - name: Upload results - uses: github/codeql-action/upload-sarif@e8893c57a1f3a2b659b6b55564fdfdbbd2982911 # v3.24.0 - with: - sarif_file: 'trivy-results.sarif' diff --git a/CMakeLists.txt b/CMakeLists.txt index e366726fe..9ccc7e0e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,6 +57,18 @@ option(UMF_FORMAT_CODE_STYLE OFF) # Only a part of skips is treated as a failure now. TODO: extend to all tests option(UMF_TESTS_FAIL_ON_SKIP "Treat skips in tests as fail" OFF) +set(UMF_TESTS_DEVDAX_PATH + "" + CACHE + PATH + "Path of the devdax used in tests. The test is skipped if it is not set or empty." +) +set(UMF_TESTS_DEVDAX_SIZE + "" + CACHE + STRING + "Size of the devdax used in tests. The test is skipped if it is not set or empty." +) option(UMF_USE_ASAN "Enable AddressSanitizer checks" OFF) option(UMF_USE_UBSAN "Enable UndefinedBehaviorSanitizer checks" OFF) option(UMF_USE_TSAN "Enable ThreadSanitizer checks" OFF) @@ -401,11 +413,20 @@ endif() if(NOT UMF_DISABLE_HWLOC) add_optional_symbol(umfOsMemoryProviderOps) + if(LINUX) + add_optional_symbol(umfDevDaxMemoryProviderOps) + endif() endif() add_subdirectory(src) if(UMF_BUILD_TESTS) + if(UMF_TESTS_DEVDAX_PATH OR UMF_TESTS_DEVDAX_SIZE) + message( + STATUS "UMF_TESTS_DEVDAX_PATH is set to: ${UMF_TESTS_DEVDAX_PATH}") + message( + STATUS "UMF_TESTS_DEVDAX_SIZE is set to: ${UMF_TESTS_DEVDAX_SIZE}") + endif() add_subdirectory(test) endif() diff --git a/README.md b/README.md index 0004f0923..9bd084bd5 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,8 @@ List of options provided by CMake: | UMF_DEVELOPER_MODE | Enable additional developer checks | ON/OFF | OFF | | UMF_FORMAT_CODE_STYLE | Add clang, cmake, and black -format-check and -format-apply targets to make | ON/OFF | OFF | | UMF_TESTS_FAIL_ON_SKIP | Treat skips in tests as fail | ON/OFF | OFF | +| UMF_TESTS_DEVDAX_PATH | Path of the devdax used in tests. The test is skipped if it is not set or empty. | path | "" | +| UMF_TESTS_DEVDAX_SIZE | Size of the devdax used in tests. The test is skipped if it is not set or empty. | size | "" | | UMF_USE_ASAN | Enable AddressSanitizer checks | ON/OFF | OFF | | UMF_USE_UBSAN | Enable UndefinedBehaviorSanitizer checks | ON/OFF | OFF | | UMF_USE_TSAN | Enable ThreadSanitizer checks | ON/OFF | OFF | diff --git a/include/umf/providers/provider_devdax_memory.h b/include/umf/providers/provider_devdax_memory.h new file mode 100644 index 000000000..b2266fa3c --- /dev/null +++ b/include/umf/providers/provider_devdax_memory.h @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#ifndef UMF_DEVDAX_MEMORY_PROVIDER_H +#define UMF_DEVDAX_MEMORY_PROVIDER_H + +#include "umf/memory_provider.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/// @cond +#define UMF_DEVDAX_RESULTS_START_FROM 2000 +/// @endcond + +/// @brief Memory provider settings struct +typedef struct umf_devdax_memory_provider_params_t { + /// path of the devdax + char *path; + /// size of the devdax + size_t size; + /// combination of 'umf_mem_protection_flags_t' flags + unsigned protection; +} umf_devdax_memory_provider_params_t; + +/// @brief Devdax Memory Provider operation results +typedef enum umf_devdax_memory_provider_native_error { + UMF_DEVDAX_RESULT_SUCCESS = UMF_DEVDAX_RESULTS_START_FROM, ///< Success + UMF_DEVDAX_RESULT_ERROR_ALLOC_FAILED, ///< Memory allocation failed + UMF_DEVDAX_RESULT_ERROR_ADDRESS_NOT_ALIGNED, ///< Allocated address is not aligned + UMF_DEVDAX_RESULT_ERROR_FREE_FAILED, ///< Memory deallocation failed + UMF_DEVDAX_RESULT_ERROR_PURGE_LAZY_FAILED, ///< Lazy purging failed + UMF_DEVDAX_RESULT_ERROR_PURGE_FORCE_FAILED, ///< Force purging failed +} umf_devdax_memory_provider_native_error_t; + +umf_memory_provider_ops_t *umfDevDaxMemoryProviderOps(void); + +/// @brief Create default params for os memory provider +static inline umf_devdax_memory_provider_params_t +umfDevDaxMemoryProviderParamsDefault(char *path, size_t size) { + umf_devdax_memory_provider_params_t params = { + path, /* path of the devdax */ + size, /* size of the devdax */ + UMF_PROTECTION_READ | UMF_PROTECTION_WRITE, /* protection */ + }; + + return params; +} + +#ifdef __cplusplus +} +#endif + +#endif /* UMF_DEVDAX_MEMORY_PROVIDER_H */ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7d2f151f0..b2586da93 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -89,6 +89,7 @@ set(UMF_SOURCES_MACOSX libumf_linux.c) set(UMF_SOURCES_WINDOWS libumf_windows.c) set(UMF_SOURCES_COMMON_LINUX_MACOSX + provider/provider_devdax_memory.c provider/provider_os_memory.c provider/provider_os_memory_posix.c memtargets/memtarget_numa.c diff --git a/src/provider/provider_devdax_memory.c b/src/provider/provider_devdax_memory.c new file mode 100644 index 000000000..efecb4345 --- /dev/null +++ b/src/provider/provider_devdax_memory.c @@ -0,0 +1,677 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include +#include +#include +#include +#include +#include +#include + +#include "base_alloc_global.h" +#include "critnib.h" +#include "provider_devdax_memory_internal.h" +#include "provider_os_memory_internal.h" +#include "utils_common.h" +#include "utils_concurrency.h" +#include "utils_log.h" + +#include +#include +#include + +#define NODESET_STR_BUF_LEN 1024 + +#define TLS_MSG_BUF_LEN 1024 + +typedef struct os_last_native_error_t { + int32_t native_error; + int errno_value; + char msg_buff[TLS_MSG_BUF_LEN]; +} os_last_native_error_t; + +static __TLS os_last_native_error_t TLS_last_native_error; + +// helper values used only in the Native_error_str array +#define _UMF_DEVDAX_RESULT_SUCCESS \ + (UMF_DEVDAX_RESULT_SUCCESS - UMF_DEVDAX_RESULT_SUCCESS) +#define _UMF_DEVDAX_RESULT_ERROR_ALLOC_FAILED \ + (UMF_DEVDAX_RESULT_ERROR_ALLOC_FAILED - UMF_DEVDAX_RESULT_SUCCESS) +#define _UMF_DEVDAX_RESULT_ERROR_ADDRESS_NOT_ALIGNED \ + (UMF_DEVDAX_RESULT_ERROR_ADDRESS_NOT_ALIGNED - UMF_DEVDAX_RESULT_SUCCESS) +#define _UMF_DEVDAX_RESULT_ERROR_FREE_FAILED \ + (UMF_DEVDAX_RESULT_ERROR_FREE_FAILED - UMF_DEVDAX_RESULT_SUCCESS) +#define _UMF_DEVDAX_RESULT_ERROR_PURGE_LAZY_FAILED \ + (UMF_DEVDAX_RESULT_ERROR_PURGE_LAZY_FAILED - UMF_DEVDAX_RESULT_SUCCESS) +#define _UMF_DEVDAX_RESULT_ERROR_PURGE_FORCE_FAILED \ + (UMF_DEVDAX_RESULT_ERROR_PURGE_FORCE_FAILED - UMF_DEVDAX_RESULT_SUCCESS) + +static const char *Native_error_str[] = { + [_UMF_DEVDAX_RESULT_SUCCESS] = "success", + [_UMF_DEVDAX_RESULT_ERROR_ALLOC_FAILED] = "memory allocation failed", + [_UMF_DEVDAX_RESULT_ERROR_ADDRESS_NOT_ALIGNED] = + "allocated address is not aligned", + [_UMF_DEVDAX_RESULT_ERROR_FREE_FAILED] = "memory deallocation failed", + [_UMF_DEVDAX_RESULT_ERROR_PURGE_LAZY_FAILED] = "lazy purging failed", + [_UMF_DEVDAX_RESULT_ERROR_PURGE_FORCE_FAILED] = "force purging failed", +}; + +static void os_store_last_native_error(int32_t native_error, int errno_value) { + TLS_last_native_error.native_error = native_error; + TLS_last_native_error.errno_value = errno_value; +} + +static int devdax_copy_path(const char *in_path, char out_path[NAME_MAX]) { + // (- 1) because there should be a room for the terminating null byte ('\0') + size_t max_len = NAME_MAX - 1; + + if (strlen(in_path) > max_len) { + LOG_ERR("path of a devdax is longer than %zu bytes", max_len); + return -1; + } + + strncpy(out_path, in_path, max_len); + out_path[NAME_MAX - 1] = '\0'; // the terminating null byte + + return 0; +} + +static umf_result_t +devdax_translate_params(umf_devdax_memory_provider_params_t *in_params, + devdax_memory_provider_t *provider) { + umf_result_t result; + + result = os_translate_mem_protection_flags(in_params->protection, + &provider->protection); + if (result != UMF_RESULT_SUCCESS) { + LOG_ERR("incorrect memory protection flags: %u", in_params->protection); + return result; + } + + /* devdax requires UMF_MEM_MAP_SHARED */ + result = os_translate_mem_visibility_flag(UMF_MEM_MAP_SHARED, + &provider->visibility); + if (result != UMF_RESULT_SUCCESS) { + LOG_ERR("incorrect memory visibility flag: %u", UMF_MEM_MAP_SHARED); + return result; + } + + return UMF_RESULT_SUCCESS; +} + +static umf_result_t devdax_initialize(void *params, void **provider) { + umf_result_t ret; + + if (provider == NULL || params == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + umf_devdax_memory_provider_params_t *in_params = + (umf_devdax_memory_provider_params_t *)params; + + if (in_params->path == NULL) { + LOG_ERR("devdax path is missing"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (in_params->size == 0) { + LOG_ERR("devdax size is 0"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + devdax_memory_provider_t *devdax_provider = + umf_ba_global_alloc(sizeof(devdax_memory_provider_t)); + if (!devdax_provider) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + memset(devdax_provider, 0, sizeof(*devdax_provider)); + + ret = devdax_translate_params(in_params, devdax_provider); + if (ret != UMF_RESULT_SUCCESS) { + goto err_free_devdax_provider; + } + + devdax_provider->fd = os_devdax_open(in_params->path); + if (devdax_provider->fd == -1) { + LOG_ERR("cannot open the devdax: %s", in_params->path); + ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; + goto err_free_devdax_provider; + } + + devdax_copy_path(in_params->path, devdax_provider->path); + devdax_provider->size = in_params->size; + + devdax_provider->base = + os_mmap(NULL, devdax_provider->size, devdax_provider->protection, + devdax_provider->visibility, devdax_provider->fd, 0); + if (devdax_provider->base == NULL) { + LOG_PDEBUG("devdax memory mapping failed (path=%s, size=%zu)", + in_params->path, devdax_provider->size); + ret = UMF_RESULT_ERROR_UNKNOWN; + goto err_free_devdax_provider; + } + + LOG_DEBUG("devdax memory mapped (path=%s, size=%zu)", in_params->path, + devdax_provider->size); + + if (util_mutex_init(&devdax_provider->lock) == NULL) { + LOG_ERR("lock init failed"); + ret = UMF_RESULT_ERROR_UNKNOWN; + goto err_free_devdax_provider; + } + + devdax_provider->fd_offset_map = critnib_new(); + if (!devdax_provider->fd_offset_map) { + LOG_ERR("creating file descriptor offset map failed"); + ret = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + goto err_free_devdax_provider; + } + + *provider = devdax_provider; + + return UMF_RESULT_SUCCESS; + +err_free_devdax_provider: + umf_ba_global_free(devdax_provider); + return ret; +} + +static void devdax_finalize(void *provider) { + if (provider == NULL) { + assert(0); + return; + } + + devdax_memory_provider_t *devdax_provider = provider; + util_mutex_destroy_not_free(&devdax_provider->lock); + os_munmap(devdax_provider->base, devdax_provider->size); + critnib_delete(devdax_provider->fd_offset_map); + umf_ba_global_free(devdax_provider); +} + +static umf_result_t devdax_get_min_page_size(void *provider, void *ptr, + size_t *page_size); + +static inline void assert_is_page_aligned(uintptr_t ptr, size_t page_size) { + assert((ptr & (page_size - 1)) == 0); + (void)ptr; // unused in Release build + (void)page_size; // unused in Release build +} + +static int devdax_alloc_aligned(size_t length, size_t alignment, + size_t page_size, int fd, void *base, + size_t size, os_mutex_t *lock, void **out_addr, + size_t *offset, size_t *old_offset) { + assert(out_addr); + assert(fd > 0); + + size_t extended_length = length; + + if (alignment > page_size) { + // We have to increase length by alignment to be able to "cut out" + // the correctly aligned part of the memory from the mapped region + // by unmapping the rest: unaligned beginning and unaligned end + // of this region. + extended_length += alignment; + } + + if (util_mutex_lock(lock)) { + LOG_ERR("locking file offset failed"); + return -1; + } + + if (*offset + extended_length > size) { + util_mutex_unlock(lock); + LOG_ERR("cannot allocate more memory than the devdax size: %zu", size); + return -1; + } + + *old_offset = *offset; + *offset += extended_length; + void *ptr = (char *)base + *old_offset; + + util_mutex_unlock(lock); + + if (alignment > page_size) { + uintptr_t addr = (uintptr_t)ptr; + uintptr_t aligned_addr = addr; + uintptr_t rest_of_div = aligned_addr % alignment; + + if (rest_of_div) { + aligned_addr += alignment - rest_of_div; + } + + assert_is_page_aligned(aligned_addr, page_size); + + // tail address has to page-aligned + uintptr_t tail = aligned_addr + length; + if (tail & (page_size - 1)) { + tail = (tail + page_size) & ~(page_size - 1); + } + + assert_is_page_aligned(tail, page_size); + assert(tail >= aligned_addr + length); + + *out_addr = (void *)aligned_addr; + return 0; + } + + *out_addr = ptr; + return 0; +} + +static umf_result_t devdax_alloc(void *provider, size_t size, size_t alignment, + void **resultPtr) { + int ret; + + if (provider == NULL || resultPtr == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + devdax_memory_provider_t *devdax_provider = + (devdax_memory_provider_t *)provider; + + size_t page_size; + umf_result_t result = devdax_get_min_page_size(provider, NULL, &page_size); + if (result != UMF_RESULT_SUCCESS) { + return result; + } + + if (alignment && (alignment % page_size) && (page_size % alignment)) { + LOG_ERR("wrong alignment: %zu (not a multiple or a divider of the " + "minimum page size (%zu))", + alignment, page_size); + + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + size_t old_offset; // needed for critnib_insert() + + void *addr = NULL; + errno = 0; + ret = devdax_alloc_aligned(size, alignment, page_size, devdax_provider->fd, + devdax_provider->base, devdax_provider->size, + &devdax_provider->lock, &addr, + &devdax_provider->offset, &old_offset); + if (ret) { + os_store_last_native_error(UMF_DEVDAX_RESULT_ERROR_ALLOC_FAILED, 0); + LOG_ERR("memory allocation failed"); + return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; + } + + // verify the alignment + if ((alignment > 0) && ((uintptr_t)addr % alignment)) { + os_store_last_native_error(UMF_DEVDAX_RESULT_ERROR_ADDRESS_NOT_ALIGNED, + 0); + LOG_ERR("allocated address 0x%llx is not aligned to %zu (0x%zx) " + "bytes", + (unsigned long long)addr, alignment, alignment); + goto err_revert; + } + + // store (offset + 1) to be able to store offset == 0 + ret = critnib_insert(devdax_provider->fd_offset_map, (uintptr_t)addr, + (void *)(uintptr_t)(old_offset + 1), 0 /* update */); + if (ret) { + LOG_ERR("devdax_alloc(): inserting a value to the file descriptor " + "offset map failed (addr=%p, offset=%zu)", + addr, old_offset); + } + + *resultPtr = addr; + + return UMF_RESULT_SUCCESS; + +err_revert: + if (util_mutex_lock(&devdax_provider->lock)) { + return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; + } + + devdax_provider->offset = old_offset; + util_mutex_unlock(&devdax_provider->lock); + + return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; +} + +// free() is not supported +static umf_result_t devdax_free(void *provider, void *ptr, size_t size) { + (void)provider; // unused + (void)ptr; // unused + (void)size; // unused + + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + +static void devdax_get_last_native_error(void *provider, const char **ppMessage, + int32_t *pError) { + (void)provider; // unused + + if (ppMessage == NULL || pError == NULL) { + assert(0); + return; + } + + *pError = TLS_last_native_error.native_error; + if (TLS_last_native_error.errno_value == 0) { + *ppMessage = Native_error_str[*pError - UMF_DEVDAX_RESULT_SUCCESS]; + return; + } + + const char *msg; + size_t len; + size_t pos = 0; + + msg = Native_error_str[*pError - UMF_DEVDAX_RESULT_SUCCESS]; + len = strlen(msg); + memcpy(TLS_last_native_error.msg_buff + pos, msg, len + 1); + pos += len; + + msg = ": "; + len = strlen(msg); + memcpy(TLS_last_native_error.msg_buff + pos, msg, len + 1); + pos += len; + + os_strerror(TLS_last_native_error.errno_value, + TLS_last_native_error.msg_buff + pos, TLS_MSG_BUF_LEN - pos); + + *ppMessage = TLS_last_native_error.msg_buff; +} + +static umf_result_t devdax_get_recommended_page_size(void *provider, + size_t size, + size_t *page_size) { + (void)size; // unused + + if (provider == NULL || page_size == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + *page_size = os_get_page_size(); + + return UMF_RESULT_SUCCESS; +} + +static umf_result_t devdax_get_min_page_size(void *provider, void *ptr, + size_t *page_size) { + (void)ptr; // unused + + return devdax_get_recommended_page_size(provider, 0, page_size); +} + +static umf_result_t devdax_purge_lazy(void *provider, void *ptr, size_t size) { + if (provider == NULL || ptr == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + errno = 0; + if (os_purge(ptr, size, UMF_PURGE_LAZY)) { + os_store_last_native_error(UMF_DEVDAX_RESULT_ERROR_PURGE_LAZY_FAILED, + errno); + LOG_PERR("lazy purging failed"); + + return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; + } + return UMF_RESULT_SUCCESS; +} + +static umf_result_t devdax_purge_force(void *provider, void *ptr, size_t size) { + if (provider == NULL || ptr == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + errno = 0; + if (os_purge(ptr, size, UMF_PURGE_FORCE)) { + os_store_last_native_error(UMF_DEVDAX_RESULT_ERROR_PURGE_FORCE_FAILED, + errno); + LOG_PERR("force purging failed"); + return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; + } + return UMF_RESULT_SUCCESS; +} + +static const char *devdax_get_name(void *provider) { + (void)provider; // unused + return "OS"; +} + +// This function is supposed to be thread-safe, so it should NOT be called concurrently +// with devdax_allocation_merge() with the same pointer. +static umf_result_t devdax_allocation_split(void *provider, void *ptr, + size_t totalSize, + size_t firstSize) { + (void)totalSize; + + devdax_memory_provider_t *devdax_provider = + (devdax_memory_provider_t *)provider; + if (devdax_provider->fd <= 0) { + return UMF_RESULT_SUCCESS; + } + + void *value = critnib_get(devdax_provider->fd_offset_map, (uintptr_t)ptr); + if (value == NULL) { + LOG_ERR("devdax_allocation_split(): getting a value from the file " + "descriptor offset map failed (addr=%p)", + ptr); + return UMF_RESULT_ERROR_UNKNOWN; + } + + uintptr_t new_key = (uintptr_t)ptr + firstSize; + void *new_value = (void *)((uintptr_t)value + firstSize); + int ret = critnib_insert(devdax_provider->fd_offset_map, new_key, new_value, + 0 /* update */); + if (ret) { + LOG_ERR("devdax_allocation_split(): inserting a value to the file " + "descriptor offset map failed (addr=%p, offset=%zu)", + (void *)new_key, (size_t)new_value - 1); + return UMF_RESULT_ERROR_UNKNOWN; + } + + return UMF_RESULT_SUCCESS; +} + +// It should NOT be called concurrently with devdax_allocation_split() with the same pointer. +static umf_result_t devdax_allocation_merge(void *provider, void *lowPtr, + void *highPtr, size_t totalSize) { + (void)lowPtr; + (void)totalSize; + + devdax_memory_provider_t *devdax_provider = + (devdax_memory_provider_t *)provider; + if (devdax_provider->fd <= 0) { + return UMF_RESULT_SUCCESS; + } + + void *value = + critnib_remove(devdax_provider->fd_offset_map, (uintptr_t)highPtr); + if (value == NULL) { + LOG_ERR("devdax_allocation_merge(): removing a value from the file " + "descriptor offset map failed (addr=%p)", + highPtr); + return UMF_RESULT_ERROR_UNKNOWN; + } + + return UMF_RESULT_SUCCESS; +} + +typedef struct devdax_ipc_data_t { + char dd_path[NAME_MAX]; // path to the /dev/dax + size_t dd_size; // size of the /dev/dax + size_t offset; // offset of the data +} devdax_ipc_data_t; + +static umf_result_t devdax_get_ipc_handle_size(void *provider, size_t *size) { + if (provider == NULL || size == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + *size = sizeof(devdax_ipc_data_t); + + return UMF_RESULT_SUCCESS; +} + +static umf_result_t devdax_get_ipc_handle(void *provider, const void *ptr, + size_t size, void *providerIpcData) { + (void)size; // unused + + if (provider == NULL || ptr == NULL || providerIpcData == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + devdax_memory_provider_t *devdax_provider = + (devdax_memory_provider_t *)provider; + if (devdax_provider->fd <= 0) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + void *value = critnib_get(devdax_provider->fd_offset_map, (uintptr_t)ptr); + if (value == NULL) { + LOG_ERR("devdax_get_ipc_handle(): getting a value from the IPC cache " + "failed (addr=%p)", + ptr); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + devdax_ipc_data_t *devdax_ipc_data = (devdax_ipc_data_t *)providerIpcData; + devdax_ipc_data->offset = (size_t)value - 1; + strncpy(devdax_ipc_data->dd_path, devdax_provider->path, NAME_MAX - 1); + devdax_ipc_data->dd_path[NAME_MAX - 1] = '\0'; + devdax_ipc_data->dd_size = devdax_provider->size; + + return UMF_RESULT_SUCCESS; +} + +static umf_result_t devdax_put_ipc_handle(void *provider, + void *providerIpcData) { + if (provider == NULL || providerIpcData == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + devdax_memory_provider_t *devdax_provider = + (devdax_memory_provider_t *)provider; + devdax_ipc_data_t *devdax_ipc_data = (devdax_ipc_data_t *)providerIpcData; + + // verify the path of the /dev/dax + if (strncmp(devdax_ipc_data->dd_path, devdax_provider->path, NAME_MAX)) { + LOG_ERR("devdax path mismatch (local: %s, ipc: %s)", + devdax_provider->path, devdax_ipc_data->dd_path); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + // verify the size of the /dev/dax + if (devdax_ipc_data->dd_size != devdax_provider->size) { + LOG_ERR("devdax size mismatch (local: %zu, ipc: %zu)", + devdax_provider->size, devdax_ipc_data->dd_size); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + return UMF_RESULT_SUCCESS; +} + +static umf_result_t devdax_open_ipc_handle(void *provider, + void *providerIpcData, void **ptr) { + if (provider == NULL || providerIpcData == NULL || ptr == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + devdax_memory_provider_t *devdax_provider = + (devdax_memory_provider_t *)provider; + devdax_ipc_data_t *devdax_ipc_data = (devdax_ipc_data_t *)providerIpcData; + + // verify it is the same devdax + // verify the path of the /dev/dax + if (strncmp(devdax_ipc_data->dd_path, devdax_provider->path, NAME_MAX)) { + LOG_ERR("devdax path mismatch (local: %s, ipc: %s)", + devdax_provider->path, devdax_ipc_data->dd_path); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + // verify the size of the /dev/dax + if (devdax_ipc_data->dd_size != devdax_provider->size) { + LOG_ERR("devdax size mismatch (local: %zu, ipc: %zu)", + devdax_provider->size, devdax_ipc_data->dd_size); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + umf_result_t ret = UMF_RESULT_SUCCESS; + int fd = os_devdax_open(devdax_provider->path); + if (fd <= 0) { + LOG_PERR("opening a devdax (%s) failed", devdax_provider->path); + return UMF_RESULT_ERROR_UNKNOWN; + } + + char *base = + os_mmap(NULL, devdax_provider->size, devdax_provider->protection, + devdax_provider->visibility, devdax_provider->fd, 0); + if (base == NULL) { + os_store_last_native_error(UMF_DEVDAX_RESULT_ERROR_ALLOC_FAILED, errno); + LOG_PERR("devdax mapping failed (path: %s, size: %zu, protection: %i, " + "visibility: %i, fd: %i)", + devdax_provider->path, devdax_provider->size, + devdax_provider->protection, devdax_provider->visibility, fd); + ret = UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; + } + + LOG_DEBUG("devdax mapping done (path: %s, size: %zu, protection: %i, " + "visibility: %i, fd: %i, offset: %zu)", + devdax_provider->path, devdax_provider->size, + devdax_provider->protection, devdax_provider->visibility, fd, + devdax_ipc_data->offset); + + (void)utils_close_fd(fd); + + *ptr = base + devdax_ipc_data->offset; + + return ret; +} + +static umf_result_t devdax_close_ipc_handle(void *provider, void *ptr, + size_t size) { + if (provider == NULL || ptr == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + devdax_memory_provider_t *devdax_provider = + (devdax_memory_provider_t *)provider; + + errno = 0; + int ret = os_munmap(devdax_provider->base, devdax_provider->size); + // ignore error when size == 0 + if (ret && (size > 0)) { + os_store_last_native_error(UMF_DEVDAX_RESULT_ERROR_FREE_FAILED, errno); + LOG_PERR("memory unmapping failed"); + + return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; + } + + return UMF_RESULT_SUCCESS; +} + +static umf_memory_provider_ops_t UMF_DEVDAX_MEMORY_PROVIDER_OPS = { + .version = UMF_VERSION_CURRENT, + .initialize = devdax_initialize, + .finalize = devdax_finalize, + .alloc = devdax_alloc, + .free = devdax_free, + .get_last_native_error = devdax_get_last_native_error, + .get_recommended_page_size = devdax_get_recommended_page_size, + .get_min_page_size = devdax_get_min_page_size, + .get_name = devdax_get_name, + .ext.purge_lazy = devdax_purge_lazy, + .ext.purge_force = devdax_purge_force, + .ext.allocation_merge = devdax_allocation_merge, + .ext.allocation_split = devdax_allocation_split, + .ipc.get_ipc_handle_size = devdax_get_ipc_handle_size, + .ipc.get_ipc_handle = devdax_get_ipc_handle, + .ipc.put_ipc_handle = devdax_put_ipc_handle, + .ipc.open_ipc_handle = devdax_open_ipc_handle, + .ipc.close_ipc_handle = devdax_close_ipc_handle}; + +umf_memory_provider_ops_t *umfDevDaxMemoryProviderOps(void) { + return &UMF_DEVDAX_MEMORY_PROVIDER_OPS; +} diff --git a/src/provider/provider_devdax_memory_internal.h b/src/provider/provider_devdax_memory_internal.h new file mode 100644 index 000000000..fd19dcf97 --- /dev/null +++ b/src/provider/provider_devdax_memory_internal.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#ifndef UMF_DEVDAX_MEMORY_PROVIDER_INTERNAL_H +#define UMF_DEVDAX_MEMORY_PROVIDER_INTERNAL_H + +#include + +#include "critnib.h" +#include "utils_concurrency.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define NAME_MAX 255 + +typedef struct devdax_memory_provider_t { + char path[NAME_MAX]; // a path to the devdax + size_t size; // size of the file used for memory mapping + int fd; // file descriptor for memory mapping + void *base; // base address of memory mapping + size_t offset; // offset in the file used for memory mapping + os_mutex_t lock; // lock of ptr and offset + + unsigned protection; // combination of OS-specific protection flags + unsigned visibility; // memory visibility mode + + // A critnib map storing (ptr, fd_offset + 1) pairs. We add 1 to fd_offset + // in order to be able to store fd_offset equal 0, because + // critnib_get() returns value or NULL, so a value cannot equal 0. + // It is needed mainly in the get_ipc_handle and open_ipc_handle hooks + // to mmap a specific part of a file. + critnib *fd_offset_map; +} devdax_memory_provider_t; + +#ifdef __cplusplus +} +#endif + +#endif /* UMF_DEVDAX_MEMORY_PROVIDER_INTERNAL_H */ diff --git a/src/provider/provider_os_memory_internal.h b/src/provider/provider_os_memory_internal.h index 81d729d27..c8dd41d99 100644 --- a/src/provider/provider_os_memory_internal.h +++ b/src/provider/provider_os_memory_internal.h @@ -100,6 +100,8 @@ size_t os_get_page_size(void); void os_strerror(int errnum, char *buf, size_t buflen); +int os_devdax_open(const char *path); + #ifdef __cplusplus } #endif diff --git a/src/provider/provider_os_memory_posix.c b/src/provider/provider_os_memory_posix.c index 9308f6a18..63f38cca3 100644 --- a/src/provider/provider_os_memory_posix.c +++ b/src/provider/provider_os_memory_posix.c @@ -6,10 +6,13 @@ */ #include +#include #include #include #include +#include #include +#include #include #include @@ -106,3 +109,39 @@ void os_strerror(int errnum, char *buf, size_t buflen) { LOG_PERR("Retrieving error code description failed"); } } + +// open a devdax +int os_devdax_open(const char *path) { + if (path == NULL) { + LOG_ERR("empty path"); + return -1; + } + + if (strstr(path, "/dev/dax") != path) { + LOG_ERR("path of the file \"%s\" does not start with \"/dev/dax\"", + path); + return -1; + } + + int fd = open(path, O_RDWR); + if (fd == -1) { + LOG_PERR("cannot open the file: %s", path); + return -1; + } + + struct stat statbuf; + int ret = stat(path, &statbuf); + if (ret) { + LOG_PERR("stat(%s) failed", path); + close(fd); + return -1; + } + + if (!S_ISCHR(statbuf.st_mode)) { + LOG_ERR("file %s is not a character device", path); + close(fd); + return -1; + } + + return fd; +} diff --git a/src/provider/provider_os_memory_windows.c b/src/provider/provider_os_memory_windows.c index 994f4d53c..b05c1118d 100644 --- a/src/provider/provider_os_memory_windows.c +++ b/src/provider/provider_os_memory_windows.c @@ -151,3 +151,10 @@ size_t os_get_page_size(void) { void os_strerror(int errnum, char *buf, size_t buflen) { strerror_s(buf, buflen, errnum); } + +// open a devdax +int os_devdax_open(const char *path) { + (void)path; // unused + + return -1; +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 59c76fe31..d69833196 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -222,6 +222,21 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented NAME memtarget SRCS memspaces/memtarget.cpp LIBS ${LIBNUMA_LIBRARIES}) + if(UMF_TESTS_DEVDAX_PATH AND UMF_TESTS_DEVDAX_SIZE) + add_umf_test( + NAME provider_devdax_memory + SRCS provider_devdax_memory.cpp + LIBS ${UMF_UTILS_FOR_TEST}) + target_compile_definitions( + umf_test-provider_devdax_memory + PRIVATE UMF_TESTS_DEVDAX_PATH="${UMF_TESTS_DEVDAX_PATH}" + UMF_TESTS_DEVDAX_SIZE=${UMF_TESTS_DEVDAX_SIZE}) + else() + message( + STATUS + "UMF_TESTS_DEVDAX_PATH or UMF_TESTS_DEVDAX_SIZE is not set - skipping the devdax memory provider test" + ) + endif() if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND UMF_BUILD_FUZZTESTS) add_subdirectory(fuzz) endif() @@ -341,6 +356,36 @@ if(LINUX) add_umf_ipc_test(TEST ipc_os_prov_anon_fd) add_umf_ipc_test(TEST ipc_os_prov_shm) endif() + if(UMF_TESTS_DEVDAX_PATH AND UMF_TESTS_DEVDAX_SIZE) + build_umf_test( + NAME + ipc_devdax_prov_consumer + SRCS + ipc_devdax_prov_consumer.c + common/ipc_common.c + common/ipc_os_prov_common.c) + build_umf_test( + NAME + ipc_devdax_prov_producer + SRCS + ipc_devdax_prov_producer.c + common/ipc_common.c + common/ipc_os_prov_common.c) + target_compile_definitions( + umf_test-ipc_devdax_prov_consumer + PRIVATE UMF_TESTS_DEVDAX_PATH="${UMF_TESTS_DEVDAX_PATH}" + UMF_TESTS_DEVDAX_SIZE=${UMF_TESTS_DEVDAX_SIZE}) + target_compile_definitions( + umf_test-ipc_devdax_prov_producer + PRIVATE UMF_TESTS_DEVDAX_PATH="${UMF_TESTS_DEVDAX_PATH}" + UMF_TESTS_DEVDAX_SIZE=${UMF_TESTS_DEVDAX_SIZE}) + add_umf_ipc_test(TEST ipc_devdax_prov) + else() + message( + STATUS + "UMF_TESTS_DEVDAX_PATH or UMF_TESTS_DEVDAX_SIZE is not set - skipping the IPC devdax memory provider test" + ) + endif() if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_LEVEL_ZERO_PROVIDER) build_umf_test( NAME diff --git a/test/ipc_devdax_prov.sh b/test/ipc_devdax_prov.sh new file mode 100755 index 000000000..db14a5ad8 --- /dev/null +++ b/test/ipc_devdax_prov.sh @@ -0,0 +1,24 @@ +# +# Copyright (C) 2024 Intel Corporation +# +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +#!/bin/bash + +set -e + +# port should be a number from the range <1024, 65535> +PORT=$(( 1024 + ( $$ % ( 65535 - 1024 )))) + +UMF_LOG_VAL="level:debug;flush:debug;output:stderr;pid:yes" + +echo "Starting ipc_os_prov_shm CONSUMER on port $PORT ..." +UMF_LOG=$UMF_LOG_VAL ./umf_test-ipc_devdax_prov_consumer $PORT & + +echo "Waiting 1 sec ..." +sleep 1 + +echo "Starting ipc_os_prov_shm PRODUCER on port $PORT ..." +UMF_LOG=$UMF_LOG_VAL ./umf_test-ipc_devdax_prov_producer $PORT diff --git a/test/ipc_devdax_prov_consumer.c b/test/ipc_devdax_prov_consumer.c new file mode 100644 index 000000000..1984b9bd7 --- /dev/null +++ b/test/ipc_devdax_prov_consumer.c @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +#include + +#include "ipc_common.h" +#include "ipc_os_prov_common.h" + +int main(int argc, char *argv[]) { + if (argc < 2) { + fprintf(stderr, "usage: %s \n", argv[0]); + return -1; + } + + int port = atoi(argv[1]); + + umf_devdax_memory_provider_params_t devdax_params = + umfDevDaxMemoryProviderParamsDefault(UMF_TESTS_DEVDAX_PATH, + UMF_TESTS_DEVDAX_SIZE); + + return run_consumer(port, umfDevDaxMemoryProviderOps(), &devdax_params, + memcopy, NULL); +} diff --git a/test/ipc_devdax_prov_producer.c b/test/ipc_devdax_prov_producer.c new file mode 100644 index 000000000..8eeb5b56c --- /dev/null +++ b/test/ipc_devdax_prov_producer.c @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +#include + +#include "ipc_common.h" +#include "ipc_os_prov_common.h" + +int main(int argc, char *argv[]) { + if (argc < 2) { + fprintf(stderr, "usage: %s \n", argv[0]); + return -1; + } + + int port = atoi(argv[1]); + + umf_devdax_memory_provider_params_t devdax_params = + umfDevDaxMemoryProviderParamsDefault(UMF_TESTS_DEVDAX_PATH, + UMF_TESTS_DEVDAX_SIZE); + + return run_producer(port, umfDevDaxMemoryProviderOps(), &devdax_params, + memcopy, NULL); +} diff --git a/test/provider_devdax_memory.cpp b/test/provider_devdax_memory.cpp new file mode 100644 index 000000000..f9f507ec2 --- /dev/null +++ b/test/provider_devdax_memory.cpp @@ -0,0 +1,249 @@ +// Copyright (C) 2023 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "base.hpp" + +#include "cpp_helpers.hpp" + +#include +#include + +using umf_test::test; + +#define INVALID_PTR ((void *)0x01) + +#define ASSERT_IS_ALIGNED(ptr, alignment) \ + ASSERT_EQ(((uintptr_t)ptr % alignment), 0) + +typedef enum purge_t { + PURGE_NONE = 0, + PURGE_LAZY = 1, + PURGE_FORCE = 2, +} purge_t; + +static const char *Native_error_str[] = { + "success", // UMF_DEVDAX_RESULT_SUCCESS + "memory allocation failed", // UMF_DEVDAX_RESULT_ERROR_ALLOC_FAILED + "allocated address is not aligned", // UMF_DEVDAX_RESULT_ERROR_ADDRESS_NOT_ALIGNED + "memory deallocation failed", // UMF_DEVDAX_RESULT_ERROR_FREE_FAILED + "lazy purging failed", // UMF_DEVDAX_RESULT_ERROR_PURGE_LAZY_FAILED + "force purging failed", // UMF_DEVDAX_RESULT_ERROR_PURGE_FORCE_FAILED +}; + +// test helpers + +static int compare_native_error_str(const char *message, int error) { + const char *error_str = Native_error_str[error - UMF_DEVDAX_RESULT_SUCCESS]; + size_t len = strlen(error_str); + return strncmp(message, error_str, len); +} + +using providerCreateExtParams = std::tuple; + +umf::provider_unique_handle_t +providerCreateExt(providerCreateExtParams params) { + umf_memory_provider_handle_t hProvider = nullptr; + auto [provider_ops, provider_params] = params; + + auto ret = + umfMemoryProviderCreate(provider_ops, provider_params, &hProvider); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + EXPECT_NE(hProvider, nullptr); + + return umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); +} + +struct umfProviderTest + : umf_test::test, + ::testing::WithParamInterface { + void SetUp() override { + test::SetUp(); + provider = providerCreateExt(this->GetParam()); + umf_result_t umf_result = + umfMemoryProviderGetMinPageSize(provider.get(), NULL, &page_size); + EXPECT_EQ(umf_result, UMF_RESULT_SUCCESS); + + page_plus_64 = page_size + 64; + } + + void TearDown() override { test::TearDown(); } + + umf::provider_unique_handle_t provider; + size_t page_size; + size_t page_plus_64; +}; + +static void test_alloc_free_success(umf_memory_provider_handle_t provider, + size_t size, size_t alignment, + purge_t purge) { + void *ptr = nullptr; + + umf_result_t umf_result = + umfMemoryProviderAlloc(provider, size, alignment, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + + memset(ptr, 0xFF, size); + + if (purge == PURGE_LAZY) { + umf_result = umfMemoryProviderPurgeLazy(provider, ptr, size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } else if (purge == PURGE_FORCE) { + umf_result = umfMemoryProviderPurgeForce(provider, ptr, size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + + umf_result = umfMemoryProviderFree(provider, ptr, size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); +} + +static void verify_last_native_error(umf_memory_provider_handle_t provider, + int32_t err) { + const char *message; + int32_t error; + umfMemoryProviderGetLastNativeError(provider, &message, &error); + ASSERT_EQ(error, err); + ASSERT_EQ(compare_native_error_str(message, error), 0); +} + +static void test_alloc_failure(umf_memory_provider_handle_t provider, + size_t size, size_t alignment, + umf_result_t result, int32_t err) { + void *ptr = nullptr; + umf_result_t umf_result = + umfMemoryProviderAlloc(provider, size, alignment, &ptr); + ASSERT_EQ(umf_result, result); + ASSERT_EQ(ptr, nullptr); + + if (umf_result == UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC) { + verify_last_native_error(provider, err); + } +} + +// TESTS + +// positive tests using test_alloc_free_success + +auto defaultParams = umfDevDaxMemoryProviderParamsDefault( + (char *)UMF_TESTS_DEVDAX_PATH, UMF_TESTS_DEVDAX_SIZE); + +INSTANTIATE_TEST_SUITE_P(devdaxProviderTest, umfProviderTest, + ::testing::Values(providerCreateExtParams{ + umfDevDaxMemoryProviderOps(), &defaultParams})); + +TEST_P(umfProviderTest, create_destroy) {} + +TEST_P(umfProviderTest, alloc_page64_align_0) { + test_alloc_free_success(provider.get(), page_plus_64, 0, PURGE_NONE); +} + +TEST_P(umfProviderTest, alloc_page64_align_page_div_2) { + test_alloc_free_success(provider.get(), page_plus_64, page_size / 2, + PURGE_NONE); +} + +TEST_P(umfProviderTest, alloc_page64_align_3_pages) { + test_alloc_free_success(provider.get(), page_plus_64, 3 * page_size, + PURGE_NONE); +} + +TEST_P(umfProviderTest, alloc_3pages_align_3pages) { + test_alloc_free_success(provider.get(), 3 * page_size, 3 * page_size, + PURGE_NONE); +} + +/* +TEST_P(umfProviderTest, purge_lazy) { + test_alloc_free_success(provider.get(), page_plus_64, 0, PURGE_LAZY); +} +*/ + +TEST_P(umfProviderTest, purge_force) { + test_alloc_free_success(provider.get(), page_plus_64, 0, PURGE_FORCE); +} + +// negative tests using test_alloc_failure + +TEST_P(umfProviderTest, alloc_page64_align_page_minus_1_WRONG_ALIGNMENT_1) { + test_alloc_failure(provider.get(), page_plus_64, page_size - 1, + UMF_RESULT_ERROR_INVALID_ARGUMENT, 0); +} + +TEST_P(umfProviderTest, alloc_page64_align_one_half_pages_WRONG_ALIGNMENT_2) { + test_alloc_failure(provider.get(), page_plus_64, + page_size + (page_size / 2), + UMF_RESULT_ERROR_INVALID_ARGUMENT, 0); +} + +TEST_P(umfProviderTest, alloc_WRONG_SIZE) { + test_alloc_failure(provider.get(), -1, 0, + UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC, + UMF_DEVDAX_RESULT_ERROR_ALLOC_FAILED); +} + +// other positive tests + +TEST_P(umfProviderTest, get_min_page_size) { + size_t min_page_size; + umf_result_t umf_result = umfMemoryProviderGetMinPageSize( + provider.get(), nullptr, &min_page_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_LE(min_page_size, page_size); +} + +TEST_P(umfProviderTest, get_recommended_page_size) { + size_t min_page_size; + umf_result_t umf_result = umfMemoryProviderGetMinPageSize( + provider.get(), nullptr, &min_page_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_LE(min_page_size, page_size); + + size_t recommended_page_size; + umf_result = umfMemoryProviderGetRecommendedPageSize( + provider.get(), 0, &recommended_page_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_GE(recommended_page_size, min_page_size); +} + +TEST_P(umfProviderTest, get_name) { + const char *name = umfMemoryProviderGetName(provider.get()); + ASSERT_STREQ(name, "OS"); +} + +TEST_P(umfProviderTest, free_size_0_ptr_not_null) { + umf_result_t umf_result = + umfMemoryProviderFree(provider.get(), INVALID_PTR, 0); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); +} + +TEST_P(umfProviderTest, free_NULL) { + umf_result_t umf_result = umfMemoryProviderFree(provider.get(), nullptr, 0); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); +} + +// other negative tests + +TEST_P(umfProviderTest, free_INVALID_POINTER_SIZE_GT_0) { + umf_result_t umf_result = + umfMemoryProviderFree(provider.get(), INVALID_PTR, page_plus_64); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); +} + +TEST_P(umfProviderTest, purge_lazy_INVALID_POINTER) { + umf_result_t umf_result = + umfMemoryProviderPurgeLazy(provider.get(), INVALID_PTR, 1); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC); + + verify_last_native_error(provider.get(), + UMF_DEVDAX_RESULT_ERROR_PURGE_LAZY_FAILED); +} + +TEST_P(umfProviderTest, purge_force_INVALID_POINTER) { + umf_result_t umf_result = + umfMemoryProviderPurgeForce(provider.get(), INVALID_PTR, 1); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC); + + verify_last_native_error(provider.get(), + UMF_DEVDAX_RESULT_ERROR_PURGE_FORCE_FAILED); +}