Fallback to hwloc fetch if package not found

PatKamin · PatKamin · commit 97d296c65625 · 2025-03-13T08:06:15.000+01:00
Fallback to fetching hwloc from source instead of failing UMF build.
Add a workflow for testing the fallback.
Unify correct paths to hwloc.
Use target name instead of a library path for hwloc linkage.
diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml
@@ -318,6 +318,56 @@ jobs:
       run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{env.BUILD_DIR}}/src/proxy_lib/${{matrix.build_type}}/umf_proxy.dll
       shell: pwsh
 
+  hwloc-fallback:
+    name: "Fallback to static hwloc build"
+    strategy:
+      matrix:
+        os: ['ubuntu-latest', 'windows-latest']
+        build_type: [Release]
+        include:
+          - os: 'ubuntu-latest'
+            build_type: Release
+            number_of_processors: '$(nproc)'
+          - os: 'windows-latest'
+            build_type: Release
+            number_of_processors: '$Env:NUMBER_OF_PROCESSORS'
+    
+    runs-on: ${{matrix.os}}
+
+    steps:
+    - name: Install dependencies
+      if: matrix.os == 'ubuntu-latest'
+      run: sudo apt-get install -y libnuma-dev
+
+    - name: Checkout
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      with:
+        fetch-depth: 0
+
+    - name: Configure build
+      run: >
+        cmake
+        -B ${{env.BUILD_DIR}}
+        -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
+        -DUMF_BUILD_SHARED_LIBRARY=ON
+        -DUMF_BUILD_EXAMPLES=OFF
+        -DUMF_DEVELOPER_MODE=ON
+        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON
+        -DUMF_BUILD_CUDA_PROVIDER=ON
+        -DUMF_LINK_HWLOC_STATICALLY=OFF
+        -DUMF_TESTS_FAIL_ON_SKIP=ON
+
+    - name: Build UMF
+      run: >
+        cmake
+        --build ${{env.BUILD_DIR}}
+        --config ${{matrix.build_type}}
+        -j ${{matrix.number_of_processors}}
+
+    - name: Run tests
+      working-directory: ${{env.BUILD_DIR}}
+      run: ctest -C ${{matrix.build_type}} --output-on-failure --test-dir test
+
   windows-dynamic_build_hwloc:
     name: "Windows dynamic UMF + static hwloc"
     strategy:
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -257,126 +257,96 @@ else()
     if(NOT UMF_LINK_HWLOC_STATICALLY)
         pkg_check_modules(LIBHWLOC hwloc>=2.3.0)
         if(NOT LIBHWLOC_FOUND)
-            find_package(LIBHWLOC 2.3.0 REQUIRED hwloc)
+            find_package(LIBHWLOC 2.3.0 COMPONENTS hwloc)
+            if(LIBHWLOC_LIBRARIES)
+                set(LIBHWLOC_AVAILABLE TRUE)
+            endif()
         endif()
 
-        # add PATH to DLL on Windows
-        set(DLL_PATH_LIST
-            "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_DLL_DIRS}")
-    elseif(WINDOWS)
-        include(FetchContent)
-        set(HWLOC_ENABLE_TESTING OFF)
-        set(HWLOC_SKIP_LSTOPO ON)
-        set(HWLOC_SKIP_TOOLS ON)
+        if(LIBHWLOC_AVAILABLE OR LIBHWLOC_FOUND)
+            # add PATH to DLL on Windows
+            set(DLL_PATH_LIST
+                "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_DLL_DIRS}")
+            message(STATUS "    LIBHWLOC_LIBRARIES = ${LIBHWLOC_LIBRARIES}")
+        else()
+            set(UMF_LINK_HWLOC_STATICALLY ON)
+        endif()
+    endif()
 
+    if(UMF_LINK_HWLOC_STATICALLY)
+        include(FetchContent)
         message(
             STATUS
                 "Will fetch hwloc from ${UMF_HWLOC_REPO} (tag: ${UMF_HWLOC_TAG})"
         )
 
-        FetchContent_Declare(
-            hwloc_targ
-            GIT_REPOSITORY ${UMF_HWLOC_REPO}
-            GIT_TAG ${UMF_HWLOC_TAG}
-            SOURCE_SUBDIR contrib/windows-cmake/ FIND_PACKAGE_ARGS)
-        FetchContent_MakeAvailable(hwloc_targ)
+        if(WINDOWS)
+            set(HWLOC_ENABLE_TESTING OFF)
+            set(HWLOC_SKIP_LSTOPO ON)
+            set(HWLOC_SKIP_TOOLS ON)
+
+            FetchContent_Declare(
+                hwloc_targ
+                GIT_REPOSITORY ${UMF_HWLOC_REPO}
+                GIT_TAG ${UMF_HWLOC_TAG}
+                SOURCE_SUBDIR contrib/windows-cmake/)
+            FetchContent_MakeAvailable(hwloc_targ)
+
+            set(HWLOC_LIB_PATH "")
+            if(CMAKE_GENERATOR STREQUAL "NMake Makefiles")
+                set(HWLOC_LIB_PATH "${hwloc_targ_BINARY_DIR}/hwloc.lib")
+            else()
+                set(HWLOC_LIB_PATH "${hwloc_targ_BINARY_DIR}/lib/hwloc.lib")
+            endif()
+
+            get_filename_component(LIBHWLOC_LIBRARY_DIRS ${HWLOC_LIB_PATH}
+                                   DIRECTORY)
+            set(LIBHWLOC_LIBRARIES ${HWLOC_LIB_PATH})
+            set(LIBHWLOC_INCLUDE_DIRS ${hwloc_targ_BINARY_DIR}/include)
+        else() # not Windows
+            FetchContent_Declare(
+                hwloc_targ
+                GIT_REPOSITORY ${UMF_HWLOC_REPO}
+                GIT_TAG ${UMF_HWLOC_TAG})
+            FetchContent_MakeAvailable(hwloc_targ)
 
-        message(STATUS "hwloc CMAKE_GENERATOR: ${CMAKE_GENERATOR}")
-
-        if(CMAKE_GENERATOR STREQUAL "Ninja" OR CMAKE_GENERATOR STREQUAL
-                                               "Unix Makefiles")
             add_custom_command(
-                COMMAND ${CMAKE_COMMAND}
-                        -DCMAKE_INSTALL_PREFIX=${hwloc_targ_BINARY_DIR} -B build
-                WORKING_DIRECTORY
-                    ${hwloc_targ_SOURCE_DIR}/contrib/windows-cmake/
-                OUTPUT
-                    ${hwloc_targ_SOURCE_DIR}/contrib/windows-cmake/CMakeCache.txt
-            )
+                COMMAND ./autogen.sh
+                WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
+                OUTPUT ${hwloc_targ_SOURCE_DIR}/configure)
             add_custom_command(
-                COMMAND ${CMAKE_COMMAND} --build build
-                WORKING_DIRECTORY
-                    ${hwloc_targ_SOURCE_DIR}/contrib/windows-cmake/
-                OUTPUT
-                    ${hwloc_targ_SOURCE_DIR}/contrib/windows-cmake/build/lib/hwloc.lib
-                DEPENDS
-                    ${hwloc_targ_SOURCE_DIR}/contrib/windows-cmake/CMakeCache.txt
-            )
+                COMMAND
+                    ./configure --prefix=${hwloc_targ_BINARY_DIR}
+                    --enable-static=yes --enable-shared=no --disable-libxml2
+                    --disable-pci --disable-levelzero --disable-opencl
+                    --disable-cuda --disable-nvml --disable-libudev
+                    --disable-rsmi CFLAGS=-fPIC CXXFLAGS=-fPIC
+                WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
+                OUTPUT ${hwloc_targ_SOURCE_DIR}/Makefile
+                DEPENDS ${hwloc_targ_SOURCE_DIR}/configure)
             add_custom_command(
-                COMMAND ${CMAKE_COMMAND} --build build --target INSTALL
-                WORKING_DIRECTORY
-                    ${hwloc_targ_SOURCE_DIR}/contrib/windows-cmake/
-                OUTPUT ${hwloc_targ_BINARY_DIR}/lib/hwloc.lib
-                DEPENDS
-                    ${hwloc_targ_SOURCE_DIR}/contrib/windows-cmake/build/lib/hwloc.lib
-            )
+                COMMAND make
+                WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
+                OUTPUT ${hwloc_targ_SOURCE_DIR}/lib/libhwloc.la
+                DEPENDS ${hwloc_targ_SOURCE_DIR}/Makefile)
+            add_custom_command(
+                COMMAND make install
+                WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
+                OUTPUT ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a
+                DEPENDS ${hwloc_targ_SOURCE_DIR}/lib/libhwloc.la)
+
             add_custom_target(hwloc_prod
-                              DEPENDS ${hwloc_targ_BINARY_DIR}/lib/hwloc.lib)
+                              DEPENDS ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a)
+            add_library(hwloc INTERFACE)
             target_link_libraries(
-                hwloc INTERFACE ${hwloc_targ_BINARY_DIR}/lib/hwloc.lib)
+                hwloc INTERFACE ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a)
             add_dependencies(hwloc hwloc_prod)
 
             set(LIBHWLOC_LIBRARY_DIRS ${hwloc_targ_BINARY_DIR}/lib)
-            set(LIBHWLOC_LIBRARIES ${hwloc_targ_BINARY_DIR}/lib/hwloc.lib)
-        elseif(CMAKE_GENERATOR STREQUAL "NMake Makefiles")
-            set(LIBHWLOC_LIBRARY_DIRS ${hwloc_targ_BINARY_DIR}/)
-            set(LIBHWLOC_LIBRARIES ${hwloc_targ_BINARY_DIR}/hwloc.lib)
-        else()
-            set(LIBHWLOC_LIBRARY_DIRS ${hwloc_targ_BINARY_DIR}/$<CONFIG>)
-            set(LIBHWLOC_LIBRARIES ${hwloc_targ_BINARY_DIR}/$<CONFIG>/hwloc.lib)
+            set(LIBHWLOC_INCLUDE_DIRS ${hwloc_targ_BINARY_DIR}/include)
+            set(LIBHWLOC_LIBRARIES ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a)
         endif()
-
-        set(LIBHWLOC_INCLUDE_DIRS
-            ${hwloc_targ_SOURCE_DIR}/include;${hwloc_targ_BINARY_DIR}/include)
-    else()
-        include(FetchContent)
-        message(
-            STATUS
-                "Will fetch hwloc from ${UMF_HWLOC_REPO} (tag: ${UMF_HWLOC_TAG})"
-        )
-
-        FetchContent_Declare(
-            hwloc_targ
-            GIT_REPOSITORY ${UMF_HWLOC_REPO}
-            GIT_TAG ${UMF_HWLOC_TAG})
-        FetchContent_MakeAvailable(hwloc_targ)
-
-        add_custom_command(
-            COMMAND ./autogen.sh
-            WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
-            OUTPUT ${hwloc_targ_SOURCE_DIR}/configure)
-        add_custom_command(
-            COMMAND
-                ./configure --prefix=${hwloc_targ_BINARY_DIR}
-                --enable-static=yes --enable-shared=no --disable-libxml2
-                --disable-pci --disable-levelzero --disable-opencl
-                --disable-cuda --disable-nvml --disable-libudev --disable-rsmi
-                CFLAGS=-fPIC CXXFLAGS=-fPIC
-            WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
-            OUTPUT ${hwloc_targ_SOURCE_DIR}/Makefile
-            DEPENDS ${hwloc_targ_SOURCE_DIR}/configure)
-        add_custom_command(
-            COMMAND make
-            WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
-            OUTPUT ${hwloc_targ_SOURCE_DIR}/lib/libhwloc.la
-            DEPENDS ${hwloc_targ_SOURCE_DIR}/Makefile)
-        add_custom_command(
-            COMMAND make install
-            WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
-            OUTPUT ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a
-            DEPENDS ${hwloc_targ_SOURCE_DIR}/lib/libhwloc.la)
-
-        add_custom_target(hwloc_prod
-                          DEPENDS ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a)
-        add_library(hwloc INTERFACE)
-        target_link_libraries(hwloc
-                              INTERFACE ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a)
-        add_dependencies(hwloc hwloc_prod)
-
-        set(LIBHWLOC_LIBRARY_DIRS ${hwloc_targ_BINARY_DIR}/lib)
-        set(LIBHWLOC_INCLUDE_DIRS ${hwloc_targ_BINARY_DIR}/include)
-        set(LIBHWLOC_LIBRARIES ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a)
-    endif()
+    endif() # UMF_LINK_HWLOC_STATICALLY
 
     message(STATUS "    LIBHWLOC_LIBRARIES = ${LIBHWLOC_LIBRARIES}")
     message(STATUS "    LIBHWLOC_INCLUDE_DIRS = ${LIBHWLOC_INCLUDE_DIRS}")
@@ -388,7 +358,7 @@ else()
 endif()
 
 if(hwloc_targ_SOURCE_DIR)
-    # apply security patch for HWLOC
+    # Apply security patch for HWLOC
     execute_process(
         COMMAND git apply ${PROJECT_SOURCE_DIR}/cmake/fix_coverity_issues.patch
         WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
diff --git a/README.md b/README.md
@@ -29,7 +29,7 @@ using the CUDA API and UMF CUDA memory provider.
 
 Required packages:
 
-- libhwloc-dev >= 2.3.0 (Linux) / hwloc >= 2.3.0 (Windows)
+- libhwloc-dev >= 2.3.0 (Linux) / hwloc >= 2.3.0 (Windows) (required for examples)
 - C compiler
 - [CMake](https://cmake.org/) >= 3.14.0
 
diff --git a/cmake/FindLIBHWLOC.cmake b/cmake/FindLIBHWLOC.cmake
@@ -55,7 +55,7 @@ if(WINDOWS)
 endif()
 
 if(LIBHWLOC_LIBRARY)
-    message(STATUS "  Found libhwloc using find_library()")
+    message(STATUS "  Found libhwloc: ${LIBHWLOC_LIBRARY}")
 
     if(LIBHWLOC_FIND_VERSION)
         if(NOT LIBHWLOC_API_VERSION)
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
@@ -18,7 +18,7 @@ set(EXAMPLE_NAME umf_example_basic)
 add_umf_executable(
     NAME ${EXAMPLE_NAME}
     SRCS basic/basic.c
-    LIBS umf ${LIBHWLOC_LIBRARIES})
+    LIBS umf ${UMF_HWLOC_NAME})
 
 target_include_directories(
     ${EXAMPLE_NAME} PRIVATE ${UMF_CMAKE_SOURCE_DIR}/src/utils
@@ -231,7 +231,7 @@ if(LINUX)
     add_umf_executable(
         NAME ${EXAMPLE_NAME}
         SRCS memspace_numa/memspace_numa.c
-        LIBS umf ${LIBHWLOC_LIBRARIES} numa)
+        LIBS umf ${UMF_HWLOC_NAME} numa)
 
     target_include_directories(
         ${EXAMPLE_NAME}
@@ -254,7 +254,7 @@ if(LINUX)
     add_umf_executable(
         NAME ${EXAMPLE_NAME}
         SRCS memspace_hmat/memspace_hmat.c
-        LIBS umf ${LIBHWLOC_LIBRARIES} numa)
+        LIBS umf ${UMF_HWLOC_NAME} numa)
 
     target_include_directories(
         ${EXAMPLE_NAME}
@@ -278,7 +278,7 @@ if(LINUX)
         add_umf_executable(
             NAME ${EXAMPLE_NAME}
             SRCS custom_file_provider/custom_file_provider.c
-            LIBS umf ${LIBHWLOC_LIBRARIES})
+            LIBS umf ${UMF_HWLOC_NAME})
 
         target_include_directories(
             ${EXAMPLE_NAME} PRIVATE ${UMF_CMAKE_SOURCE_DIR}/src/utils
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -103,7 +103,7 @@ endif()
 if(NOT UMF_DISABLE_HWLOC)
     set(UMF_SOURCES ${UMF_SOURCES} ${HWLOC_DEPENDENT_SOURCES}
                     memtargets/memtarget_numa.c)
-    set(UMF_LIBS ${UMF_LIBS} ${LIBHWLOC_LIBRARIES})
+    set(UMF_LIBS ${UMF_LIBS} $<BUILD_INTERFACE:${UMF_HWLOC_NAME}>)
     set(UMF_PRIVATE_LIBRARY_DIRS ${UMF_PRIVATE_LIBRARY_DIRS}
                                  ${LIBHWLOC_LIBRARY_DIRS})
 else()
@@ -150,14 +150,11 @@ if(UMF_BUILD_SHARED_LIBRARY)
         set(CMAKE_INSTALL_RPATH "${UMF_INSTALL_RPATH}")
     endif()
 
-    if(NOT UMF_DISABLE_HWLOC)
-        set(HWLOC_LIB ${UMF_HWLOC_NAME})
-    endif()
     add_umf_library(
         NAME umf
         TYPE SHARED
         SRCS ${UMF_SOURCES}
-        LIBS ${UMF_LIBS} ${HWLOC_LIB}
+        LIBS ${UMF_LIBS}
         LINUX_MAP_FILE ${CMAKE_CURRENT_SOURCE_DIR}/libumf.map
         WINDOWS_DEF_FILE ${CMAKE_CURRENT_SOURCE_DIR}/libumf.def)
     set(UMF_COMMON_COMPILE_DEFINITIONS ${UMF_COMMON_COMPILE_DEFINITIONS}
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -294,27 +294,27 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented
     add_umf_test(
         NAME memspace_numa
         SRCS memspaces/memspace_numa.cpp
-        LIBS ${LIBNUMA_LIBRARIES} ${LIBHWLOC_LIBRARIES})
+        LIBS ${LIBNUMA_LIBRARIES} ${UMF_HWLOC_NAME})
     add_umf_test(
         NAME provider_os_memory_config
         SRCS provider_os_memory_config.cpp
-        LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES} ${LIBHWLOC_LIBRARIES})
+        LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES} ${UMF_HWLOC_NAME})
     add_umf_test(
         NAME memspace_host_all
         SRCS memspaces/memspace_host_all.cpp
-        LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES} ${LIBHWLOC_LIBRARIES})
+        LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES} ${UMF_HWLOC_NAME})
     add_umf_test(
         NAME memspace_highest_capacity
         SRCS memspaces/memspace_highest_capacity.cpp
-        LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES} ${LIBHWLOC_LIBRARIES})
+        LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES} ${UMF_HWLOC_NAME})
     add_umf_test(
         NAME memspace_highest_bandwidth
         SRCS memspaces/memspace_highest_bandwidth.cpp
-        LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES} ${LIBHWLOC_LIBRARIES})
+        LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES} ${UMF_HWLOC_NAME})
     add_umf_test(
         NAME memspace_lowest_latency
         SRCS memspaces/memspace_lowest_latency.cpp
-        LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES} ${LIBHWLOC_LIBRARIES})
+        LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES} ${UMF_HWLOC_NAME})
     add_umf_test(
         NAME mempolicy
         SRCS memspaces/mempolicy.cpp
@@ -326,7 +326,7 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented
     add_umf_test(
         NAME memtarget
         SRCS memspaces/memtarget.cpp
-        LIBS ${LIBNUMA_LIBRARIES} ${LIBHWLOC_LIBRARIES})
+        LIBS ${LIBNUMA_LIBRARIES} ${UMF_HWLOC_NAME})
     add_umf_test(
         NAME provider_devdax_memory
         SRCS provider_devdax_memory.cpp
@@ -678,6 +678,7 @@ endif()
 # replace test_examples.sh with CMake script?)
 if(LINUX
    AND UMF_BUILD_SHARED_LIBRARY
+   AND UMF_BUILD_EXAMPLES
    AND NOT
        (UMF_USE_ASAN
         OR UMF_USE_UBSAN
