patch libLLVM/libMLIR rpaths

Author: makslevental

.clang-format

@@ -1,3 +1,2 @@
 BasedOnStyle: LLVM
 AlwaysBreakTemplateDeclarations: Yes
-ReflowComments: false

.github/workflows/build_test_release_eudsl.yml

@@ -124,6 +124,7 @@ jobs:
           $python3_command -m cibuildwheel "$PWD/projects/eudsl-tblgen" --output-dir wheelhouse
 
       - name: "Build eudsl-py"
+        if: ${{ ! startsWith(matrix.os, 'windows') }}
         run: |
           $python3_command -m cibuildwheel "$PWD/projects/eudsl-py" --output-dir wheelhouse
 
@@ -239,11 +240,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        runs-on: ["ubuntu-22.04", "macos-14", "macos-13", "windows-2019"]
+        runs-on: ["ubuntu-22.04", "macos-14", "macos-13",
+                  # "windows-2019"
+        ]
         python-version: ["3.9", "3.10", "3.11", "3.12"]
         include: [
           {runs-on: "ubuntu-22.04", name: "ubuntu_x86_64", os: "ubuntu"},
-          {runs-on: "windows-2019", name: "windows_x86_64", os: "windows"},
+          # {runs-on: "windows-2019", name: "windows_x86_64", os: "windows"},
           {runs-on: "macos-14", name: "macos_arm64", os: "macos"},
           {runs-on: "macos-13", name: "macos_x86_64", os: "macos"},
         ]
@@ -269,7 +272,7 @@ jobs:
 
       - uses: actions/download-artifact@v4
         with:
-          name: eudsl_py_${{ matrix.name }}_artifact
+          name: eudsl_${{ matrix.name }}_artifact
           path: wheelhouse
 
       - name: "Install eudsl-py"

build_tools/build_llvm.sh

@@ -45,6 +45,17 @@ cmake_options=(
   -DCMAKE_INSTALL_PREFIX="${LLVM_INSTALL_DIR}"
 )
 
+if [ -x "$(command -v ccache)" ]; then
+  echo 'using ccache' >&2
+  export CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros
+  export CCACHE_CPP2=true
+  export CCACHE_UMASK=002
+  cmake_options+=(
+    -DCMAKE_C_COMPILER_LAUNCHER=ccache
+    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+  )
+fi
+
 # last so that C/CXX flags get set first
 cmake_options+=(-C "$TD/cmake/llvm_cache.cmake")
 

build_tools/cmake/llvm_cache.cmake

@@ -14,8 +14,8 @@ set(LLVM_BUILD_LLVM_DYLIB ON CACHE BOOL "")
 # (but doesn't work on windows)
 if (NOT WIN32)
   set(LLVM_LINK_LLVM_DYLIB ON CACHE BOOL "")
+  set(MLIR_LINK_MLIR_DYLIB ON CACHE BOOL "")
 endif()
-set(MLIR_LINK_MLIR_DYLIB OFF CACHE BOOL "")
 
 # useful things
 set(LLVM_ENABLE_ASSERTIONS ON CACHE BOOL "")
@@ -103,7 +103,6 @@ set(LLVM_MlirDevelopment_DISTRIBUTION_COMPONENTS
 
     FileCheck
     not
-    MLIR
     MLIRPythonModules
     # triggers MLIRMlirDevelopmentTargets.cmake
     mlir-mlirdevelopment-cmake-exports

projects/eudsl-py/src/CMakeLists.txt renamed to projects/eudsl-py/CMakeLists.txt

@@ -2,8 +2,6 @@ cmake_minimum_required(VERSION 3.29)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
-include(AddEUDSL)
 set(LLVM_SUBPROJECT_TITLE "EUDSLPY")
 set(EUDSLPY_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
 
@@ -81,10 +79,12 @@ if(NOT TARGET LLVMSupport)
   message(FATAL_ERROR "LLVMSupport not found")
 endif()
 
+set(EUDSLPY_DIR "${CMAKE_CURRENT_LIST_DIR}/cmake")
+find_package(EUDSLPY CONFIG REQUIRED)
 add_eudslpygen(eudslpy-gen EUDSLPY
   DESTINATION "${CMAKE_INSTALL_BINDIR}"
   EXPORT ${LLVM_SUBPROJECT_TITLE}
-  eudslpy-gen.cpp
+  src/eudslpy-gen.cpp
 )
 
 eudslpygen(EUDSLGenArith
@@ -93,8 +93,6 @@ eudslpygen(EUDSLGenArith
   NAMESPACES mlir::arith mlir::arith::detail
 )
 
-
-
 execute_process(
   COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir
   OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE nanobind_DIR)
@@ -112,34 +110,70 @@ set(nanobind_options
   $<$<PLATFORM_ID:Windows>:/EHsc /GR>
 )
 
+set(EUDSLPY_SRC_DIR "${CMAKE_CURRENT_LIST_DIR}/src")
+
 nanobind_add_module(eudslpy_ext
   NB_STATIC
   NB_DOMAIN eudsl
-  eudslpy_ext.cpp
+  src/eudslpy_ext.cpp
 )
 set_target_properties(eudslpy_ext
   PROPERTIES
-  LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/eudsl"
+  LIBRARY_OUTPUT_DIRECTORY "${EUDSLPY_SRC_DIR}/eudsl"
 )
 target_link_libraries(eudslpy_ext PRIVATE MLIR)
+
+# hack so we can move libMLIR and libLLVM into the wheel
+# see AddLLVM.cmake#llvm_setup_rpath
+if(APPLE OR UNIX)
+  set(_origin_prefix "\$ORIGIN")
+  if(APPLE)
+    set(_origin_prefix "@loader_path")
+  endif()
+  if (EUDSLPY_STANDALONE_BUILD)
+    get_target_property(_mlir_loc MLIR LOCATION)
+    get_target_property(_llvm_loc LLVM LOCATION)
+  else()
+    set(_mlir_loc "$<TARGET_FILE:MLIR>")
+    set(_llvm_loc "$<TARGET_FILE:LLVM>")
+  endif()
+  set(_old_rpath "${_origin_prefix}/../lib${LLVM_LIBDIR_SUFFIX}")
+  if(APPLE)
+    execute_process(COMMAND install_name_tool -rpath "${_old_rpath}" ${_origin_prefix} "${_mlir_loc}" ERROR_VARIABLE rpath_err)
+    execute_process(COMMAND install_name_tool -rpath "${_old_rpath}" ${_origin_prefix} "${_llvm_loc}" ERROR_VARIABLE rpath_err)
+    # maybe already updated...
+    if (rpath_err AND NOT rpath_err MATCHES "no LC_RPATH load command with path: ${_old_rpath}")
+      message(FATAL_ERROR "couldn't update rpath because: ${rpath_err}")
+    endif()
+  else()
+    # sneaky sneaky - undocumented
+    file(RPATH_CHANGE FILE "${_mlir_loc}" OLD_RPATH "${_old_rpath}" NEW_RPATH "${_origin_prefix}")
+    file(RPATH_CHANGE FILE "${_llvm_loc}" OLD_RPATH "${_old_rpath}" NEW_RPATH "${_origin_prefix}")
+  endif()
+  set_target_properties(eudslpy_ext PROPERTIES INSTALL_RPATH "${_origin_prefix}")
+endif()
+# copy libMLIR into the ext dir for wheels
+install(IMPORTED_RUNTIME_ARTIFACTS MLIR LLVM LIBRARY DESTINATION eudsl)
+
 target_compile_options(eudslpy_ext PRIVATE ${nanobind_options})
 target_compile_options(nanobind-static PRIVATE ${nanobind_options})
 add_dependencies(eudslpy_ext EUDSLGenArith)
 
-set(NB_STUBGEN_CMD "${Python_EXECUTABLE}" "-m" "nanobind.stubgen" -m eudsl -r)
-set(NB_STUBGEN_OUTPUTS ${CMAKE_CURRENT_LIST_DIR}/eudsl/__init__.pyi)
+set(NB_STUBGEN_CMD "${Python_EXECUTABLE}" "-m" "nanobind.stubgen" --module eudslpy_ext --recursive --include-private --output-dir .)
+set(NB_STUBGEN_OUTPUTS "eudslpy_ext.pyi")
 add_custom_command(
   OUTPUT ${NB_STUBGEN_OUTPUTS}
   COMMAND ${NB_STUBGEN_CMD}
-  WORKING_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}"
+  WORKING_DIRECTORY "${EUDSLPY_SRC_DIR}/eudsl"
   DEPENDS eudslpy_ext
 )
 add_custom_target(eudslpy_ext_stub ALL DEPENDS ${NB_STUBGEN_OUTPUTS})
 
 install(TARGETS eudslpy_ext LIBRARY DESTINATION eudsl)
 install(
-  DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/eudsl
+  DIRECTORY "${EUDSLPY_SRC_DIR}/eudsl"
   DESTINATION ${CMAKE_INSTALL_PREFIX}
+  PATTERN "*.pyc" EXCLUDE
   PATTERN "*.so" EXCLUDE
   PATTERN "*.a" EXCLUDE
   PATTERN ".gitignore" EXCLUDE

projects/eudsl-py/src/cmake/AddEUDSL.cmake renamed to projects/eudsl-py/cmake/EUDSLPYConfig.cmake

@@ -53,34 +53,30 @@ function(eudslpygen target inputFile outputFileName)
   # this could be smarter by asking people to list td targets or something but that's too onerous
   file(GLOB_RECURSE global_tds "${MLIR_INCLUDE_DIR}/mlir/*.td")
   # use cc -MM  to collect all transitive headers
+  set(_depfile ${CMAKE_CURRENT_BINARY_DIR}/${outputFileName}.d)
   file(RELATIVE_PATH outputFileName_rel
-    ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/${outputFileName})
-  set(clang_command ${CMAKE_CXX_COMPILER} -xc++ "-std=c++${CMAKE_CXX_STANDARD}" -MM ${EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE}
-    ${eudslpygen_includes} -MT ${outputFileName_rel} -o ${outputFileName_rel}.d)
+    "${CMAKE_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/${outputFileName}")
+  set(clang_command ${CMAKE_CXX_COMPILER} -v -xc++ "-std=c++${CMAKE_CXX_STANDARD}" -MM ${EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE}
+    ${eudslpygen_includes} -MT ${outputFileName_rel} -o ${_depfile})
   execute_process(COMMAND ${clang_command} RESULT_VARIABLE had_error COMMAND_ECHO STDERR)
-  if(had_error OR NOT EXISTS "${outputFileName_rel}.d")
-    message(WARNING "couldn't build depfile")
+  if(had_error OR NOT EXISTS "${_depfile}")
+    set(additional_cmdline -o "${outputFileName_rel}")
   else()
     # Use depfile instead of globbing arbitrary *.td(s) for Ninja.
     if(CMAKE_GENERATOR MATCHES "Ninja")
-      # Make output path relative to build.ninja, assuming located on
-      # ${CMAKE_BINARY_DIR}.
+      # Make output path relative to build.ninja, assuming located on ${CMAKE_BINARY_DIR}.
       # CMake emits build targets as relative paths but Ninja doesn't identify
       # absolute path (in *.d) as relative path (in build.ninja)
       # Note that eudslpygen is executed on ${CMAKE_BINARY_DIR} as working directory.
-      set(additional_cmdline
-        -o ${outputFileName_rel}
-        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
-        DEPFILE ${CMAKE_CURRENT_BINARY_DIR}/${outputFileName}.d
-      )
+      set(additional_cmdline -o "${outputFileName_rel}" DEPFILE "${_depfile}")
     else()
       # the length of the first line in the depfile...
       string(LENGTH "${outputFileName_rel}: \\" depfile_offset)
-      file(READ ${CMAKE_CURRENT_BINARY_DIR}/${outputFileName}.d local_headers OFFSET ${depfile_offset})
+      file(READ ${_depfile} local_headers OFFSET ${depfile_offset})
       string(REPLACE "\\" ";" local_headers "${local_headers}")
       string(REGEX REPLACE "[ \t\r\n]" "" local_headers "${local_headers}")
       list(REMOVE_ITEM local_headers "")
-      set(additional_cmdline -o ${CMAKE_CURRENT_BINARY_DIR}/${outputFileName})
+      set(additional_cmdline -o "${outputFileName_rel}")
     endif()
   endif()
 
@@ -112,6 +108,7 @@ function(eudslpygen target inputFile outputFileName)
     ${eudslpygen_includes}
     ${eudslpygen_defines}
     ${additional_cmdline}
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
     # The file in EUDSLPYGEN_TARGET_DEFINITIONS may be not in the current
     # directory and local_headers may not contain it, so we must
     # explicitly list it here:

projects/eudsl-py/pyproject.toml

@@ -22,12 +22,13 @@ Homepage = "https://github.com/llvm/eudsl"
 minimum-version = "0.4"
 build-dir = "build/{wheel_tag}"
 wheel.py-api = "cp312"
-cmake.source-dir = "src"
+cmake.source-dir = "."
 
 [tool.scikit-build.cmake.define]
 LLVM_DIR = {env="LLVM_DIR", default="EMPTY"}
 MLIR_DIR = {env="MLIR_DIR", default="EMPTY"}
 Clang_DIR = {env="Clang_DIR", default="EMPTY"}
+CMAKE_PREFIX_PATH = {env="CMAKE_PREFIX_PATH", default="EMPTY"}
 CMAKE_CXX_VISIBILITY_PRESET = "hidden"
 CMAKE_VERBOSE_MAKEFILE = "ON"
 
@@ -40,5 +41,6 @@ environment-pass = [
     "LLVM_DIR",
     "MLIR_DIR",
     "Clang_DIR",
-    "CMAKE_GENERATOR"
+    "CMAKE_GENERATOR",
+    "CMAKE_PREFIX_PATH"
 ]