[core] Generalize TROOT::GetLibDir()

The problem with `TROOT::GetLibDir()` was that it either hardcodes the
install directory inside it if `ROOTLIBDIR` was set (which is the case
for `gnuinstall=ON`), or it assumes that it's always called `lib` and is
inside the `$ROOTSYS`, which might or might not be set. This is broken
if the `CMAKE_INSTALL_LIBDIR` is different from the default `lib`.

This commit suggests to automatically figure out the ROOT library
directory instead.

Closes the following JIRA ticket:
https://its.cern.ch/jira/browse/ROOT-9569

Author: guitargeek

core/base/CMakeLists.txt

@@ -207,6 +207,21 @@ target_include_directories(Core PUBLIC
   $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/inc>
 )
 
+if(ROOT_NEED_STDCXXFS)
+    target_link_libraries(Core PRIVATE stdc++fs)
+endif()
+
+# This code about the LIB_CORE_NAME define is important for TROOT::GetLibDir()
+get_target_property(core_prefix Core PREFIX)
+get_target_property(core_suffix Core SUFFIX)
+get_target_property(core_soversion Core SOVERSION)
+if(core_soversion)
+    set(full_core_filename "${core_prefix}Core${core_suffix}.${core_soversion}")
+else()
+    set(full_core_filename "${core_prefix}Core${core_suffix}")
+endif()
+target_compile_options(Core PRIVATE -DLIB_CORE_NAME=${full_core_filename})
+
 if(PCRE2_FOUND)
   target_link_libraries(Core PRIVATE PCRE2::PCRE2)
   set_source_files_properties(src/TPRegexp.cxx

core/base/src/TROOT.cxx

@@ -71,6 +71,7 @@ of a main program creating an interactive version is shown below:
 #include <ROOT/RVersion.hxx>
 #include "RConfigure.h"
 #include "RConfigOptions.h"
+#include <filesystem>
 #include <string>
 #include <map>
 #include <set>
@@ -109,8 +110,12 @@ FARPROC dlsym(void *library, const char *function_name)
       return ::GetProcAddress((HMODULE)library, function_name);
    }
 }
+#elif defined(__APPLE__)
+#include <dlfcn.h>
+#include <mach-o/dyld.h>
 #else
 #include <dlfcn.h>
+#include <link.h>
 #endif
 
 #include <iostream>
@@ -3007,23 +3012,71 @@ const TString& TROOT::GetBinDir() {
 
 ////////////////////////////////////////////////////////////////////////////////
 /// Get the library directory in the installation. Static utility function.
+///
+/// This function inspects the libraries currently loaded in the process to
+/// locate the ROOT Core library. Once found, it extracts and returns the
+/// directory containing that library. If the ROOT Core library was not found,
+/// it will return an empty string.
+///
+/// The result is cached in a static variable so the lookup is only performed
+/// once per process, and the implementation is platform-specific.
+///
+/// \return The directory path (as a `TString`) containing the ROOT core library.
 
-const TString& TROOT::GetLibDir() {
-#ifdef ROOTLIBDIR
-   if (IgnorePrefix()) {
-#endif
-      static TString rootlibdir;
-      if (rootlibdir.IsNull()) {
-         rootlibdir = "lib";
-         gSystem->PrependPathName(GetRootSys(), rootlibdir);
-      }
-      return rootlibdir;
-#ifdef ROOTLIBDIR
-   } else {
-      const static TString rootlibdir = ROOTLIBDIR;
+const TString &TROOT::GetLibDir()
+{
+   static bool haveLooked = false;
+   static TString rootlibdir;
+   if (haveLooked)
       return rootlibdir;
+
+   haveLooked = true;
+
+   namespace fs = std::filesystem;
+
+#define TO_LITERAL(string) _QUOTE_(string)
+
+#if defined(__APPLE__)
+
+   uint32_t count = _dyld_image_count();
+   for (uint32_t i = 0; i < count; i++) {
+      const char *path = _dyld_get_image_name(i);
+      if (!path)
+         continue;
+
+      fs::path p(path);
+      if (p.filename() == TO_LITERAL(LIB_CORE_NAME)) {
+         rootlibdir = p.parent_path().c_str();
+         break;
+      }
    }
+
+#elif defined(_WIN32)
+
+   // Or Windows, the original hardcoded path is kept for now.
+   rootlibdir = "lib";
+   gSystem->PrependPathName(GetRootSys(), rootlibdir);
+
+#else
+
+   auto callback = +[](struct dl_phdr_info *info, size_t /*size*/, void *data) -> int {
+      TString &libdir = *static_cast<TString *>(data);
+      if (!info->dlpi_name)
+         return 0;
+
+      fs::path p = info->dlpi_name;
+      if (p.filename() == TO_LITERAL(LIB_CORE_NAME)) {
+         libdir = p.parent_path().c_str();
+         return 1; // stop iteration
+      }
+      return 0; // continue
+   };
+
+   dl_iterate_phdr(callback, &rootlibdir);
+
 #endif
+
+   return rootlibdir;
 }
 
 ////////////////////////////////////////////////////////////////////////////////