[CMake] Set rpath for individual targets and not globally

When using the `rpath=ON` option, the global CMake variable
`CMAKE_INSTALL_RPATH` is mutated, which is problematic in several corner
cases.

This commit suggests to append the rpaths at the target property level,
and also set the rpaths for the build tree separately so they make
sense.

A new ROOT macro is introduced for this rpath setting, which are also
used for the PyROOT libraries, which also need custom rpaths.

Closes #19327 and #19134.

Author: guitargeek

cmake/modules/RootBuildOptions.cmake

@@ -427,23 +427,6 @@ include(RootInstallDirs)
 # add to RPATH any directories outside the project that are in the linker search path
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
-if(rpath)
-  file(RELATIVE_PATH BINDIR_TO_LIBDIR "${CMAKE_INSTALL_FULL_BINDIR}" "${CMAKE_INSTALL_FULL_LIBDIR}")
-
-  if(APPLE)
-    set(CMAKE_MACOSX_RPATH TRUE)
-    set(CMAKE_INSTALL_NAME_DIR "@rpath")
-
-    set(_rpath_values "@loader_path" "@loader_path/${BINDIR_TO_LIBDIR}")
-  else()
-    set(_rpath_values "$ORIGIN" "$ORIGIN/${BINDIR_TO_LIBDIR}")
-  endif()
-
-  set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH-CACHED};${_rpath_values}" CACHE STRING "Install RPATH" FORCE)
-
-  unset(BINDIR_TO_LIBDIR)
-endif()
-
 #---deal with the DCMAKE_IGNORE_PATH------------------------------------------------------------
 if(macos_native)
   if(APPLE)

cmake/modules/RootMacros.cmake

@@ -1028,6 +1028,9 @@ function(ROOT_LINKER_LIBRARY library)
 
   #----Installation details-------------------------------------------------------
   if(NOT ARG_TEST AND NOT ARG_NOINSTALL AND CMAKE_LIBRARY_OUTPUT_DIRECTORY)
+    if(rpath)
+      ROOT_APPEND_LIBDIR_TO_INSTALL_RPATH(${library} ${CMAKE_INSTALL_LIBDIR})
+    endif()
     if(ARG_CMAKENOEXPORT)
       install(TARGETS ${library} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT libraries
                                  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT libraries
@@ -1515,6 +1518,9 @@ function(ROOT_EXECUTABLE executable)
   endif()
   #----Installation details------------------------------------------------------
   if(NOT ARG_NOINSTALL AND CMAKE_RUNTIME_OUTPUT_DIRECTORY)
+    if(rpath)
+      ROOT_APPEND_LIBDIR_TO_INSTALL_RPATH(${executable} ${CMAKE_INSTALL_BINDIR})
+    endif()
     if(ARG_CMAKENOEXPORT)
       install(TARGETS ${executable} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT applications)
     else()
@@ -2150,3 +2156,28 @@ function(generateManual name input output)
 
   install(FILES ${output} DESTINATION ${CMAKE_INSTALL_MANDIR}/man1)
 endfunction()
+
+#----------------------------------------------------------------------------
+# --- ROOT_APPEND_LIBDIR_TO_INSTALL_RPATH(target install_dir)
+#
+# Sets the INSTALL_RPATH for a given target so that it can find the ROOT shared
+# libraries at runtime. The RPATH is set relative to the target's own location
+# using $ORIGIN (or @loader_path on macOS).
+#
+# Arguments:
+#   target       - The CMake target (e.g., a shared library or executable)
+#   install_dir  - The install subdirectory relative to CMAKE_INSTALL_PREFIX
+#----------------------------------------------------------------------------
+function(ROOT_APPEND_LIBDIR_TO_INSTALL_RPATH target install_dir)
+  file(RELATIVE_PATH to_libdir "${CMAKE_INSTALL_PREFIX}/${install_dir}" "${CMAKE_INSTALL_FULL_LIBDIR}")
+
+  # New path
+  if(APPLE)
+    set(new_rpath "@loader_path/${to_libdir}")
+  else()
+    set(new_rpath "$ORIGIN/${to_libdir}")
+  endif()
+
+  # Append to existing RPATH
+  set_property(TARGET ${target} APPEND PROPERTY INSTALL_RPATH "${new_rpath}")
+endfunction()