Fallback to hwloc fetch if package not found

PatKamin · PatKamin · commit 42daf3986356 · 2025-03-17T09:14:46.000+01:00
Fallback to fetching hwloc from source instead of failing UMF build.
Add a workflow for testing the fallback.
Unify correct paths to hwloc.
Use target name instead of a library path for hwloc linkage.
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -10,6 +10,9 @@ on:
 permissions:
   contents: read
 
+env:
+  BUILD_DIR : "${{github.workspace}}/build"
+
 jobs:
   fuzz-test:
       name: Fuzz test
@@ -92,7 +95,6 @@ jobs:
     name: Windows-Ninja-cl
     env:
       VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows"
-      BUILD_DIR : "${{github.workspace}}/build"
     strategy:
       matrix:
         os: ['windows-2019', 'windows-2022']
@@ -155,7 +157,6 @@ jobs:
     name: Windows-NMake
     env:
       VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows"
-      BUILD_DIR : "${{github.workspace}}/build"
     strategy:
       matrix:
         os: ['windows-2019', 'windows-2022']
@@ -215,7 +216,6 @@ jobs:
    name: ICX
    env:
      VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows"
-     BUILD_DIR : "${{github.workspace}}/build"
    strategy:
      matrix:
        os: ['windows-2019', 'windows-2022']
@@ -297,6 +297,56 @@ jobs:
        call "C:\Program Files (x86)\Intel\oneAPI\setvars-vcvarsall.bat"
        ctest -C ${{matrix.build_type}} --output-on-failure --test-dir test
 
+  hwloc-fallback:
+  # Scenarios where UMF_LINK_HWLOC_STATICALLY is set to OFF and hwloc is not installed in the system
+  # The hwloc library is fetched implicitly
+    name: "Fallback to static hwloc build"
+    strategy:
+      matrix:
+        os: ['ubuntu-latest', 'windows-latest']
+        build_type: [Release]
+        include:
+          - os: 'ubuntu-latest'
+            build_type: Release
+            number_of_processors: '$(nproc)'
+          - os: 'windows-latest'
+            build_type: Release
+            number_of_processors: '$Env:NUMBER_OF_PROCESSORS'
+    
+    runs-on: ${{matrix.os}}
+
+    steps:
+    - name: Install dependencies
+      if: matrix.os == 'ubuntu-latest'
+      run: sudo apt-get install -y libnuma-dev
+
+    - name: Checkout
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      with:
+        fetch-depth: 0
+
+    - name: Configure build
+      run: >
+        cmake
+        -B ${{env.BUILD_DIR}}
+        -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
+        -DUMF_BUILD_SHARED_LIBRARY=ON
+        -DUMF_BUILD_EXAMPLES=OFF
+        -DUMF_DEVELOPER_MODE=ON
+        -DUMF_LINK_HWLOC_STATICALLY=OFF
+        -DUMF_TESTS_FAIL_ON_SKIP=ON
+
+    - name: Build UMF
+      run: >
+        cmake
+        --build ${{env.BUILD_DIR}}
+        --config ${{matrix.build_type}}
+        -j ${{matrix.number_of_processors}}
+
+    - name: Run tests
+      working-directory: ${{env.BUILD_DIR}}
+      run: ctest -C ${{matrix.build_type}} --output-on-failure --test-dir test
+
   L0:
     uses: ./.github/workflows/reusable_gpu.yml
     with:
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -41,6 +41,7 @@ endif()
 include(CTest)
 include(CMakePackageConfigHelpers)
 include(GNUInstallDirs)
+include(FetchContent)
 find_package(PkgConfig)
 
 # --------------------------------------------------------------------------- #
@@ -162,7 +163,6 @@ else()
         set(UMF_JEMALLOC_TAG 5.3.0)
     endif()
 
-    include(FetchContent)
     message(
         STATUS
             "Will fetch jemalloc from ${UMF_JEMALLOC_REPO} (tag: ${UMF_JEMALLOC_TAG})"
@@ -257,126 +257,95 @@ else()
     if(NOT UMF_LINK_HWLOC_STATICALLY)
         pkg_check_modules(LIBHWLOC hwloc>=2.3.0)
         if(NOT LIBHWLOC_FOUND)
-            find_package(LIBHWLOC 2.3.0 REQUIRED hwloc)
+            find_package(LIBHWLOC 2.3.0 COMPONENTS hwloc)
+            if(LIBHWLOC_LIBRARIES)
+                set(LIBHWLOC_AVAILABLE TRUE)
+            endif()
         endif()
 
-        # add PATH to DLL on Windows
-        set(DLL_PATH_LIST
-            "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_DLL_DIRS}")
-    elseif(WINDOWS)
-        include(FetchContent)
-        set(HWLOC_ENABLE_TESTING OFF)
-        set(HWLOC_SKIP_LSTOPO ON)
-        set(HWLOC_SKIP_TOOLS ON)
+        if(LIBHWLOC_AVAILABLE OR LIBHWLOC_FOUND)
+            # add PATH to DLL on Windows
+            set(DLL_PATH_LIST
+                "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_DLL_DIRS}")
+            message(STATUS "    LIBHWLOC_LIBRARIES = ${LIBHWLOC_LIBRARIES}")
+        else()
+            set(UMF_LINK_HWLOC_STATICALLY ON)
+        endif()
+    endif()
 
+    if(UMF_LINK_HWLOC_STATICALLY)
         message(
             STATUS
                 "Will fetch hwloc from ${UMF_HWLOC_REPO} (tag: ${UMF_HWLOC_TAG})"
         )
 
-        FetchContent_Declare(
-            hwloc_targ
-            GIT_REPOSITORY ${UMF_HWLOC_REPO}
-            GIT_TAG ${UMF_HWLOC_TAG}
-            SOURCE_SUBDIR contrib/windows-cmake/ FIND_PACKAGE_ARGS)
-        FetchContent_MakeAvailable(hwloc_targ)
-
-        message(STATUS "hwloc CMAKE_GENERATOR: ${CMAKE_GENERATOR}")
+        if(WINDOWS)
+            set(HWLOC_ENABLE_TESTING OFF)
+            set(HWLOC_SKIP_LSTOPO ON)
+            set(HWLOC_SKIP_TOOLS ON)
+
+            FetchContent_Declare(
+                hwloc_targ
+                GIT_REPOSITORY ${UMF_HWLOC_REPO}
+                GIT_TAG ${UMF_HWLOC_TAG}
+                SOURCE_SUBDIR contrib/windows-cmake/)
+            FetchContent_MakeAvailable(hwloc_targ)
+
+            set(HWLOC_LIB_PATH "")
+            if(CMAKE_GENERATOR STREQUAL "NMake Makefiles")
+                set(HWLOC_LIB_PATH "${hwloc_targ_BINARY_DIR}/hwloc.lib")
+            else()
+                set(HWLOC_LIB_PATH "${hwloc_targ_BINARY_DIR}/lib/hwloc.lib")
+            endif()
+
+            get_filename_component(LIBHWLOC_LIBRARY_DIRS ${HWLOC_LIB_PATH}
+                                   DIRECTORY)
+            set(LIBHWLOC_LIBRARIES ${HWLOC_LIB_PATH})
+            set(LIBHWLOC_INCLUDE_DIRS ${hwloc_targ_BINARY_DIR}/include)
+        else() # not Windows
+            FetchContent_Declare(
+                hwloc_targ
+                GIT_REPOSITORY ${UMF_HWLOC_REPO}
+                GIT_TAG ${UMF_HWLOC_TAG})
+            FetchContent_MakeAvailable(hwloc_targ)
 
-        if(CMAKE_GENERATOR STREQUAL "Ninja" OR CMAKE_GENERATOR STREQUAL
-                                               "Unix Makefiles")
             add_custom_command(
-                COMMAND ${CMAKE_COMMAND}
-                        -DCMAKE_INSTALL_PREFIX=${hwloc_targ_BINARY_DIR} -B build
-                WORKING_DIRECTORY
-                    ${hwloc_targ_SOURCE_DIR}/contrib/windows-cmake/
-                OUTPUT
-                    ${hwloc_targ_SOURCE_DIR}/contrib/windows-cmake/CMakeCache.txt
-            )
+                COMMAND ./autogen.sh
+                WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
+                OUTPUT ${hwloc_targ_SOURCE_DIR}/configure)
             add_custom_command(
-                COMMAND ${CMAKE_COMMAND} --build build
-                WORKING_DIRECTORY
-                    ${hwloc_targ_SOURCE_DIR}/contrib/windows-cmake/
-                OUTPUT
-                    ${hwloc_targ_SOURCE_DIR}/contrib/windows-cmake/build/lib/hwloc.lib
-                DEPENDS
-                    ${hwloc_targ_SOURCE_DIR}/contrib/windows-cmake/CMakeCache.txt
-            )
+                COMMAND
+                    ./configure --prefix=${hwloc_targ_BINARY_DIR}
+                    --enable-static=yes --enable-shared=no --disable-libxml2
+                    --disable-pci --disable-levelzero --disable-opencl
+                    --disable-cuda --disable-nvml --disable-libudev
+                    --disable-rsmi CFLAGS=-fPIC CXXFLAGS=-fPIC
+                WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
+                OUTPUT ${hwloc_targ_SOURCE_DIR}/Makefile
+                DEPENDS ${hwloc_targ_SOURCE_DIR}/configure)
             add_custom_command(
-                COMMAND ${CMAKE_COMMAND} --build build --target INSTALL
-                WORKING_DIRECTORY
-                    ${hwloc_targ_SOURCE_DIR}/contrib/windows-cmake/
-                OUTPUT ${hwloc_targ_BINARY_DIR}/lib/hwloc.lib
-                DEPENDS
-                    ${hwloc_targ_SOURCE_DIR}/contrib/windows-cmake/build/lib/hwloc.lib
-            )
+                COMMAND make
+                WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
+                OUTPUT ${hwloc_targ_SOURCE_DIR}/lib/libhwloc.la
+                DEPENDS ${hwloc_targ_SOURCE_DIR}/Makefile)
+            add_custom_command(
+                COMMAND make install
+                WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
+                OUTPUT ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a
+                DEPENDS ${hwloc_targ_SOURCE_DIR}/lib/libhwloc.la)
+
             add_custom_target(hwloc_prod
-                              DEPENDS ${hwloc_targ_BINARY_DIR}/lib/hwloc.lib)
+                              DEPENDS ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a)
+            add_library(hwloc INTERFACE)
             target_link_libraries(
-                hwloc INTERFACE ${hwloc_targ_BINARY_DIR}/lib/hwloc.lib)
+                hwloc INTERFACE ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a)
             add_dependencies(hwloc hwloc_prod)
 
             set(LIBHWLOC_LIBRARY_DIRS ${hwloc_targ_BINARY_DIR}/lib)
-            set(LIBHWLOC_LIBRARIES ${hwloc_targ_BINARY_DIR}/lib/hwloc.lib)
-        elseif(CMAKE_GENERATOR STREQUAL "NMake Makefiles")
-            set(LIBHWLOC_LIBRARY_DIRS ${hwloc_targ_BINARY_DIR}/)
-            set(LIBHWLOC_LIBRARIES ${hwloc_targ_BINARY_DIR}/hwloc.lib)
-        else()
-            set(LIBHWLOC_LIBRARY_DIRS ${hwloc_targ_BINARY_DIR}/$<CONFIG>)
-            set(LIBHWLOC_LIBRARIES ${hwloc_targ_BINARY_DIR}/$<CONFIG>/hwloc.lib)
+            set(LIBHWLOC_INCLUDE_DIRS ${hwloc_targ_BINARY_DIR}/include)
+            set(LIBHWLOC_LIBRARIES ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a)
         endif()
-
-        set(LIBHWLOC_INCLUDE_DIRS
-            ${hwloc_targ_SOURCE_DIR}/include;${hwloc_targ_BINARY_DIR}/include)
-    else()
-        include(FetchContent)
-        message(
-            STATUS
-                "Will fetch hwloc from ${UMF_HWLOC_REPO} (tag: ${UMF_HWLOC_TAG})"
-        )
-
-        FetchContent_Declare(
-            hwloc_targ
-            GIT_REPOSITORY ${UMF_HWLOC_REPO}
-            GIT_TAG ${UMF_HWLOC_TAG})
-        FetchContent_MakeAvailable(hwloc_targ)
-
-        add_custom_command(
-            COMMAND ./autogen.sh
-            WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
-            OUTPUT ${hwloc_targ_SOURCE_DIR}/configure)
-        add_custom_command(
-            COMMAND
-                ./configure --prefix=${hwloc_targ_BINARY_DIR}
-                --enable-static=yes --enable-shared=no --disable-libxml2
-                --disable-pci --disable-levelzero --disable-opencl
-                --disable-cuda --disable-nvml --disable-libudev --disable-rsmi
-                CFLAGS=-fPIC CXXFLAGS=-fPIC
-            WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
-            OUTPUT ${hwloc_targ_SOURCE_DIR}/Makefile
-            DEPENDS ${hwloc_targ_SOURCE_DIR}/configure)
-        add_custom_command(
-            COMMAND make
-            WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
-            OUTPUT ${hwloc_targ_SOURCE_DIR}/lib/libhwloc.la
-            DEPENDS ${hwloc_targ_SOURCE_DIR}/Makefile)
-        add_custom_command(
-            COMMAND make install
-            WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
-            OUTPUT ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a
-            DEPENDS ${hwloc_targ_SOURCE_DIR}/lib/libhwloc.la)
-
-        add_custom_target(hwloc_prod
-                          DEPENDS ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a)
-        add_library(hwloc INTERFACE)
-        target_link_libraries(hwloc
-                              INTERFACE ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a)
-        add_dependencies(hwloc hwloc_prod)
-
-        set(LIBHWLOC_LIBRARY_DIRS ${hwloc_targ_BINARY_DIR}/lib)
-        set(LIBHWLOC_INCLUDE_DIRS ${hwloc_targ_BINARY_DIR}/include)
-        set(LIBHWLOC_LIBRARIES ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a)
-    endif()
+    endif() # UMF_LINK_HWLOC_STATICALLY
 
     message(STATUS "    LIBHWLOC_LIBRARIES = ${LIBHWLOC_LIBRARIES}")
     message(STATUS "    LIBHWLOC_INCLUDE_DIRS = ${LIBHWLOC_INCLUDE_DIRS}")
@@ -388,7 +357,7 @@ else()
 endif()
 
 if(hwloc_targ_SOURCE_DIR)
-    # apply security patch for HWLOC
+    # Apply security patch for HWLOC
     execute_process(
         COMMAND git apply ${PROJECT_SOURCE_DIR}/cmake/fix_coverity_issues.patch
         WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR}
@@ -406,8 +375,6 @@ endif()
 # Fetch L0 loader only if needed i.e.: if building L0 provider is ON and L0
 # headers are not provided by the user (via setting UMF_LEVEL_ZERO_INCLUDE_DIR).
 if(UMF_BUILD_LEVEL_ZERO_PROVIDER AND (NOT UMF_LEVEL_ZERO_INCLUDE_DIR))
-    include(FetchContent)
-
     set(LEVEL_ZERO_LOADER_REPO "https://github.com/oneapi-src/level-zero.git")
     set(LEVEL_ZERO_LOADER_TAG v1.20.2)
 
@@ -436,8 +403,6 @@ endif()
 # Fetch CUDA only if needed i.e.: if building CUDA provider is ON and CUDA
 # headers are not provided by the user (via setting UMF_CUDA_INCLUDE_DIR).
 if(UMF_BUILD_CUDA_PROVIDER AND (NOT UMF_CUDA_INCLUDE_DIR))
-    include(FetchContent)
-
     set(CUDA_REPO
         "https://gitlab.com/nvidia/headers/cuda-individual/cudart.git")
     set(CUDA_TAG cuda-12.5.1)
diff --git a/cmake/FindLIBHWLOC.cmake b/cmake/FindLIBHWLOC.cmake
@@ -55,7 +55,7 @@ if(WINDOWS)
 endif()
 
 if(LIBHWLOC_LIBRARY)
-    message(STATUS "  Found libhwloc using find_library()")
+    message(STATUS "  Found libhwloc: ${LIBHWLOC_LIBRARY}")
 
     if(LIBHWLOC_FIND_VERSION)
         if(NOT LIBHWLOC_API_VERSION)
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
@@ -18,7 +18,7 @@ set(EXAMPLE_NAME umf_example_basic)
 add_umf_executable(
     NAME ${EXAMPLE_NAME}
     SRCS basic/basic.c
-    LIBS umf ${LIBHWLOC_LIBRARIES})
+    LIBS umf ${UMF_HWLOC_NAME})
 
 target_include_directories(
     ${EXAMPLE_NAME} PRIVATE ${UMF_CMAKE_SOURCE_DIR}/src/utils
@@ -231,7 +231,7 @@ if(LINUX)
     add_umf_executable(
         NAME ${EXAMPLE_NAME}
         SRCS memspace_numa/memspace_numa.c
-        LIBS umf ${LIBHWLOC_LIBRARIES} numa)
+        LIBS umf ${UMF_HWLOC_NAME} numa)
 
     target_include_directories(
         ${EXAMPLE_NAME}
@@ -254,7 +254,7 @@ if(LINUX)
     add_umf_executable(
         NAME ${EXAMPLE_NAME}
         SRCS memspace_hmat/memspace_hmat.c
-        LIBS umf ${LIBHWLOC_LIBRARIES} numa)
+        LIBS umf ${UMF_HWLOC_NAME} numa)
 
     target_include_directories(
         ${EXAMPLE_NAME}
@@ -278,7 +278,7 @@ if(LINUX)
         add_umf_executable(
             NAME ${EXAMPLE_NAME}
             SRCS custom_file_provider/custom_file_provider.c
-            LIBS umf ${LIBHWLOC_LIBRARIES})
+            LIBS umf ${UMF_HWLOC_NAME})
 
         target_include_directories(
             ${EXAMPLE_NAME} PRIVATE ${UMF_CMAKE_SOURCE_DIR}/src/utils
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
