Merge branch 'main' into addDocker

Author: rbanka1

.cmake-format

@@ -9,31 +9,31 @@ with section("parse"):
       "pargs": 0,
       "flags": [],
       'kwargs': {
-        'NAME': '*', 
-        'SRCS': '*', 
-        'LIBS': '*' , 
+        'NAME': '*',
+        'SRCS': '*',
+        'LIBS': '*' ,
         'LIBDIRS': '*'}},
     'add_umf_executable': {
       "pargs": 0,
       "flags": [],
       'kwargs': {
-        'NAME': '*', 
-        'SRCS': '*', 
+        'NAME': '*',
+        'SRCS': '*',
         'LIBS': '*'}},
     'add_umf_test': {
       "pargs": 0,
       "flags": [],
       'kwargs': {
-        'NAME': '*', 
-        'SRCS': '*', 
+        'NAME': '*',
+        'SRCS': '*',
         'LIBS': '*'}},
     'add_umf_library': {
       "pargs": 0,
       "flags": [],
       'kwargs': {
-        'NAME': '*', 
-        'TYPE': '*', 
-        'SRCS': '*', 
+        'NAME': '*',
+        'TYPE': '*',
+        'SRCS': '*',
         'LIBS': '*',
         'LINUX_MAP_FILE': '*',
         'WINDOWS_DEF_FILE': '*'}},
@@ -43,7 +43,20 @@ with section("parse"):
       'kwargs': {
         'LABELS': '*', 
         'PASS_REGULAR_EXPRESSION': '*'}},
-  }    
+    'build_umf_test': {
+      "pargs": 0,
+      "flags": [],
+      'kwargs': {
+        'NAME': '*',
+        'SRCS': '*',
+        'LIBS': '*' }},
+    'add_umf_ipc_test': {
+      "pargs": 0,
+      "flags": [],
+      'kwargs': {
+        'TEST': '*',
+        'SRC_DIR': '*'}},
+  }
 
   # Override configurations per-command where available
   override_spec = {}

.github/workflows/reusable_basic.yml

@@ -145,9 +145,6 @@ jobs:
     - name: Install libhwloc
       run: .github/scripts/install_hwloc.sh
 
-    - name: Set ptrace value for IPC test
-      run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"
-
     - name: Get UMF version
       run: |
         VERSION=$(git describe --tags --abbrev=0 | grep -oP '\d+\.\d+\.\d+')
@@ -341,7 +338,7 @@ jobs:
         -B ${{env.BUILD_DIR}}
         -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}"
         -DUMF_BUILD_SHARED_LIBRARY=ON
-        -DUMF_BUILD_EXAMPLES=ON
+        -DUMF_BUILD_EXAMPLES=OFF
         -DUMF_FORMAT_CODE_STYLE=OFF
         -DUMF_DEVELOPER_MODE=ON
         -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
@@ -384,7 +381,7 @@ jobs:
         -B ${{env.BUILD_DIR}}
         -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}"
         -DUMF_BUILD_SHARED_LIBRARY=OFF
-        -DUMF_BUILD_EXAMPLES=ON
+        -DUMF_BUILD_EXAMPLES=OFF
         -DUMF_FORMAT_CODE_STYLE=OFF
         -DUMF_DEVELOPER_MODE=ON
         -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
@@ -456,6 +453,9 @@ jobs:
     strategy:
       matrix:
         os: ['macos-13', 'macos-14']
+        include:
+          - os: macos-14
+            static_hwloc: '-DUMF_LINK_HWLOC_STATICALLY=ON'
     env:
       BUILD_TYPE : "Release"
     runs-on: ${{matrix.os}}
@@ -474,8 +474,12 @@ jobs:
         echo "$PATH" >> $GITHUB_PATH
         python3 -m pip install -r third_party/requirements.txt
 
+    - name: Install dependencies
+      run: brew install jemalloc tbb automake libtool
+
     - name: Install hwloc
-      run: brew install hwloc tbb automake
+      if: ${{ !matrix.static_hwloc }}
+      run: brew install hwloc
 
     - name: Get UMF version
       run: |
@@ -495,6 +499,7 @@ jobs:
         -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
         -DUMF_BUILD_SHARED_LIBRARY=ON
         -DUMF_TESTS_FAIL_ON_SKIP=ON
+        ${{matrix.static_hwloc}}
 
     - name: Build UMF
       run: cmake --build ${{env.BUILD_DIR}} -j $(sysctl -n hw.logicalcpu)

.github/workflows/reusable_proxy_lib.yml

@@ -34,9 +34,6 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y cmake libhwloc-dev libtbb-dev lcov
 
-      - name: Set ptrace value for IPC test
-        run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"
-
       - name: Configure build
         run: >
           cmake

.github/workflows/reusable_sanitizers.yml

@@ -40,10 +40,6 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y intel-oneapi-ippcp-devel intel-oneapi-ipp-devel intel-oneapi-common-oneapi-vars intel-oneapi-compiler-dpcpp-cpp
 
-
-    - name: Set ptrace value for IPC test
-      run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"
-
     - name: Configure build
       run: >
         ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh &&' || ''}} 

CMakeLists.txt

@@ -10,6 +10,10 @@ list(APPEND CMAKE_MODULE_PATH "${UMF_CMAKE_SOURCE_DIR}/cmake")
 # Use full path of the helpers module (to omit potential conflicts with others)
 include(${UMF_CMAKE_SOURCE_DIR}/cmake/helpers.cmake)
 
+# --------------------------------------------------------------------------- #
+# Set UMF version variables, define project, and add basic modules
+# --------------------------------------------------------------------------- #
+
 # We use semver aligned version, set via git tags. We parse git output to
 # establish the version of UMF to be used in CMake, Win dll's, and within the
 # code (e.g. in logger). We have 3-component releases (e.g. 1.5.1) plus release
@@ -22,6 +26,12 @@ project(
     umf
     VERSION ${UMF_CMAKE_VERSION}
     LANGUAGES C)
+if(UMF_CMAKE_VERSION VERSION_EQUAL "0.0.0")
+    message(
+        WARNING
+            "UMF version is set to 0.0.0, which most likely is not expected! "
+            "Please checkout the git tags to get a proper version.")
+endif()
 
 if(PROJECT_VERSION_PATCH GREATER 0)
     # set extra variable for Windows dll metadata
@@ -33,6 +43,10 @@ include(CMakePackageConfigHelpers)
 include(GNUInstallDirs)
 find_package(PkgConfig)
 
+# --------------------------------------------------------------------------- #
+# Set UMF build options (and CACHE variables)
+# --------------------------------------------------------------------------- #
+
 # Define a list to store the names of all options
 set(UMF_OPTIONS_LIST "")
 list(APPEND UMF_OPTIONS_LIST CMAKE_BUILD_TYPE)
@@ -43,7 +57,6 @@ macro(umf_option)
     option(${ARGV})
 endmacro()
 
-# Build Options
 umf_option(UMF_BUILD_SHARED_LIBRARY "Build UMF as shared library" OFF)
 umf_option(UMF_BUILD_LEVEL_ZERO_PROVIDER "Build Level Zero memory provider" ON)
 umf_option(UMF_BUILD_CUDA_PROVIDER "Build CUDA memory provider" ON)
@@ -56,9 +69,8 @@ umf_option(UMF_BUILD_GPU_TESTS "Build UMF GPU tests" OFF)
 umf_option(UMF_BUILD_BENCHMARKS "Build UMF benchmarks" OFF)
 umf_option(UMF_BUILD_BENCHMARKS_MT "Build UMF multithreaded benchmarks" OFF)
 umf_option(UMF_BUILD_EXAMPLES "Build UMF examples" ON)
-umf_option(UMF_BUILD_FUZZTESTS "Build UMF fuzz tests" OFF)
 umf_option(UMF_BUILD_GPU_EXAMPLES "Build UMF GPU examples" OFF)
-umf_option(UMF_DEVELOPER_MODE "Enable additional developer checks" OFF)
+umf_option(UMF_BUILD_FUZZTESTS "Build UMF fuzz tests" OFF)
 umf_option(
     UMF_DISABLE_HWLOC
     "Disable hwloc and UMF features requiring it (OS provider, memtargets, topology discovery)"
@@ -67,9 +79,6 @@ umf_option(
     UMF_LINK_HWLOC_STATICALLY
     "Link UMF with HWLOC library statically (proxy library will be disabled on Windows+Debug build)"
     OFF)
-umf_option(
-    UMF_FORMAT_CODE_STYLE
-    "Add clang, cmake, and black -format-check and -format-apply targets" OFF)
 set(UMF_HWLOC_NAME
     "hwloc"
     CACHE STRING "Custom name for hwloc library w/o extension")
@@ -81,6 +90,10 @@ set(UMF_INSTALL_RPATH
         "Set the runtime search path to the directory with dependencies (e.g. hwloc)"
 )
 
+umf_option(UMF_DEVELOPER_MODE "Enable additional developer checks" OFF)
+umf_option(
+    UMF_FORMAT_CODE_STYLE
+    "Add clang, cmake, and black -format-check and -format-apply targets" OFF)
 # Only a part of skips is treated as a failure now. TODO: extend to all tests
 umf_option(UMF_TESTS_FAIL_ON_SKIP "Treat skips in tests as fail" OFF)
 umf_option(UMF_USE_ASAN "Enable AddressSanitizer checks" OFF)
@@ -100,6 +113,11 @@ set_property(CACHE UMF_PROXY_LIB_BASED_ON_POOL
              PROPERTY STRINGS ${KNOWN_PROXY_LIB_POOLS})
 list(APPEND UMF_OPTIONS_LIST UMF_PROXY_LIB_BASED_ON_POOL)
 
+# --------------------------------------------------------------------------- #
+# Setup required variables, definitions; fetch dependencies; include
+# sub_directories based on build options; set flags; etc.
+# --------------------------------------------------------------------------- #
+
 if(UMF_BUILD_TESTS
    AND DEFINED ENV{CI}
    AND NOT UMF_TESTS_FAIL_ON_SKIP)
@@ -263,8 +281,8 @@ else()
 
         set(LIBHWLOC_INCLUDE_DIRS
             ${hwloc_targ_SOURCE_DIR}/include;${hwloc_targ_BINARY_DIR}/include)
-        set(LIBHWLOC_LIBRARY_DIRS ${hwloc_targ_BINARY_DIR}/$<CONFIG>)
-        set(LIBHWLOC_LIBRARIES ${hwloc_targ_BINARY_DIR}/$<CONFIG>/hwloc.lib)
+        set(LIBHWLOC_LIBRARY_DIRS
+            ${hwloc_targ_BINARY_DIR}/Release;${hwloc_targ_BINARY_DIR}/Debug)
     else()
         include(FetchContent)
         message(
@@ -711,7 +729,7 @@ if(UMF_FORMAT_CODE_STYLE)
         add_custom_target(
             cmake-format-apply
             COMMAND ${CMAKE_FORMAT} --in-place ${format_cmake_list}
-            COMMENT "Format Cmake files using cmake-format")
+            COMMENT "Format CMake files using cmake-format")
     endif()
 
     if(BLACK)

README.md

@@ -159,11 +159,12 @@ OS memory provider supports two types of memory mappings (set by the `visibility
 IPC API requires the `UMF_MEM_MAP_SHARED` memory `visibility` mode
 (`UMF_RESULT_ERROR_INVALID_ARGUMENT` is returned otherwise).
 
-IPC API uses the file descriptor duplication. It requires using `pidfd_getfd(2)` to obtain
-a duplicate of another process's file descriptor (`pidfd_getfd(2)` is supported since Linux 5.6).
-Permission to duplicate another process's file descriptor is governed by a ptrace access mode
-`PTRACE_MODE_ATTACH_REALCREDS` check (see `ptrace(2)`) that can be changed using
-the `/proc/sys/kernel/yama/ptrace_scope` interface in the following way:
+IPC API uses file descriptor duplication, which requires the `pidfd_getfd(2)` system call to obtain
+a duplicate of another process's file descriptor. This system call is supported since Linux 5.6.
+Required permission ("restricted ptrace") is governed by the `PTRACE_MODE_ATTACH_REALCREDS` check
+(see `ptrace(2)`). To allow file descriptor duplication in a binary that opens IPC handle, you can call
+`prctl(PR_SET_PTRACER, ...)` in the producer binary that gets the IPC handle.
+Alternatively you can change the `ptrace_scope` globally in the system, e.g.:
 
 ```sh
 sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"
@@ -194,16 +195,16 @@ Packages required for tests (Linux-only yet):
 
 A memory provider that provides memory from L0 device.
 
-IPC API uses the file descriptor duplication. It requires using `pidfd_getfd(2)` to obtain
-a duplicate of another process's file descriptor (`pidfd_getfd(2)` is supported since Linux 5.6).
-Permission to duplicate another process's file descriptor is governed by a ptrace access mode
-`PTRACE_MODE_ATTACH_REALCREDS` check (see `ptrace(2)`) that can be changed using
-the `/proc/sys/kernel/yama/ptrace_scope` interface in the following way:
+IPC API uses file descriptor duplication, which requires the `pidfd_getfd(2)` system call to obtain
+a duplicate of another process's file descriptor. This system call is supported since Linux 5.6.
+Required permission ("restricted ptrace") is governed by the `PTRACE_MODE_ATTACH_REALCREDS` check
+(see `ptrace(2)`). To allow file descriptor duplication in a binary that opens IPC handle, you can call
+`prctl(PR_SET_PTRACER, ...)` in the producer binary that gets the IPC handle.
+Alternatively you can change the `ptrace_scope` globally in the system, e.g.:
 
 ```sh
 sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"
 ```
-
 ##### Requirements
 
 1) Linux or Windows OS
@@ -359,7 +360,7 @@ The memory used by the proxy memory allocator is mmap'ed:
 1) with the `MAP_PRIVATE` flag by default or
 2) with the `MAP_SHARED` flag if the `UMF_PROXY` environment variable contains one of two following strings: `page.disposition=shared-shm` or `page.disposition=shared-fd`. These two options differ in a mechanism used during IPC:
    - `page.disposition=shared-shm` - IPC uses the named shared memory. An SHM name is generated using the `umf_proxy_lib_shm_pid_$PID` pattern, where `$PID` is the PID of the process. It creates the `/dev/shm/umf_proxy_lib_shm_pid_$PID` file.
-   - `page.disposition=shared-fd` - IPC uses the file descriptor duplication. It requires using `pidfd_getfd(2)` to obtain a duplicate of another process's file descriptor. Permission to duplicate another process's file descriptor is governed by a ptrace access mode `PTRACE_MODE_ATTACH_REALCREDS` check (see `ptrace(2)`) that can be changed using the `/proc/sys/kernel/yama/ptrace_scope` interface. `pidfd_getfd(2)` is supported since Linux 5.6.
+   - `page.disposition=shared-fd` -  IPC API uses file descriptor duplication, which requires the `pidfd_getfd(2)` system call to obtain a duplicate of another process's file descriptor. This system call is supported since Linux 5.6. Required permission ("restricted ptrace") is governed by the `PTRACE_MODE_ATTACH_REALCREDS` check (see `ptrace(2)`). To allow file descriptor duplication in a binary that opens IPC handle, you can call `prctl(PR_SET_PTRACER, ...)` in the producer binary that gets the IPC handle. Alternatively you can change the `ptrace_scope` globally in the system, e.g.: `sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"`.
 
 **Size threshold**
 

RELEASE_STEPS.md

@@ -49,6 +49,11 @@ Do changes for a release:
   - If stable branch for this release is required, create it:
     - `git checkout -b v$VER.x`
     - For some early versions (like `0.1.0`) we may omit creation of the branch
+- For major/minor release, when release is done, add an extra "dev" tag on the `main` branch:
+  - `git tag -a -s -m "Development version $VERSION+1" v$VERSION+1-dev`
+    - for example, when `v0.1.0` is released, the dev tag would be `v0.2.0-dev`
+    - if needed, further in time, an extra dev tag can be introduced, e.g. `v0.2.0-dev1`
+  - This way, the `main` branch will introduce itself as the next version
 
 ## Publish changes