[core] Generalize TROOT::GetLibDir()

The problem with `TROOT::GetLibDir()` was that it either hardcodes the
install directory inside it if `ROOTLIBDIR` was set (which is the case
for `gnuinstall=ON`), or it assumes that it's always called `lib` and is
inside the `$ROOTSYS`, which might or might not be set. This is broken
if the `CMAKE_INSTALL_LIBDIR` is different from the default `lib`.

This commit suggests to automatically figure out the ROOT library
directory instead.

Closes the following JIRA ticket:
https://its.cern.ch/jira/browse/ROOT-9569

Author: guitargeek

core/base/src/TROOT.cxx

@@ -71,6 +71,7 @@ of a main program creating an interactive version is shown below:
 #include <ROOT/RVersion.hxx>
 #include "RConfigure.h"
 #include "RConfigOptions.h"
+#include <filesystem>
 #include <string>
 #include <map>
 #include <set>
@@ -109,7 +110,10 @@ FARPROC dlsym(void *library, const char *function_name)
       return ::GetProcAddress((HMODULE)library, function_name);
    }
 }
+#elif defined(__APPLE__)
+#include <mach-o/dyld.h>
 #else
+#include <link.h>
 #include <dlfcn.h>
 #endif
 
@@ -3007,23 +3011,86 @@ const TString& TROOT::GetBinDir() {
 
 ////////////////////////////////////////////////////////////////////////////////
 /// Get the library directory in the installation. Static utility function.
+///
+/// This function inspects the libraries currently loaded in the process
+/// to locate the ROOT core library (`libCore`). Once found, it extracts
+/// and returns the directory containing that library.
+///
+/// The implementation is platform-specific:
+///   - **Linux**: Uses `dl_iterate_phdr()` to iterate over all loaded
+///     ELF objects. When `libCore.so` is found, its parent directory is returned.
+///   - **macOS**: Iterates over all Mach-O images via `_dyld_image_count()`
+///     and `_dyld_get_image_name()`, looking for `libCore.dylib`
+///     (or `libCore.so` for compatibility).
+///   - **Windows**: Uses `EnumProcessModules()` and `GetModuleFileNameA()`
+///     to inspect loaded DLLs, looking for `libCore.dll`.
+///
+/// The result is cached in a static variable so the lookup is only
+/// performed once per process.
+///
+/// \return The directory path (as a `TString`) containing the ROOT core library.
 
-const TString& TROOT::GetLibDir() {
-#ifdef ROOTLIBDIR
-   if (IgnorePrefix()) {
-#endif
-      static TString rootlibdir;
-      if (rootlibdir.IsNull()) {
-         rootlibdir = "lib";
-         gSystem->PrependPathName(GetRootSys(), rootlibdir);
-      }
-      return rootlibdir;
-#ifdef ROOTLIBDIR
-   } else {
-      const static TString rootlibdir = ROOTLIBDIR;
+const TString &TROOT::GetLibDir()
+{
+   static TString rootlibdir;
+   if (!rootlibdir.IsNull())
       return rootlibdir;
+
+   namespace fs = std::filesystem;
+
+#ifdef __linux__
+
+   auto callback = +[](struct dl_phdr_info *info, size_t /*size*/, void *data) -> int {
+      TString &libdir = *static_cast<TString *>(data);
+      if (!info->dlpi_name)
+         return 0;
+
+      fs::path p = info->dlpi_name;
+      if (p.filename() == "libCore.so") {
+         libdir = p.parent_path().c_str();
+         return 1; // stop iteration
+      }
+      return 0; // continue
+   };
+
+   dl_iterate_phdr(callback, &rootlibdir);
+
+#elif defined(__APPLE__)
+
+   uint32_t count = _dyld_image_count();
+   for (uint32_t i = 0; i < count; i++) {
+      const char *path = _dyld_get_image_name(i);
+      if (!path)
+         continue;
+
+      fs::path p(path);
+      if (p.filename() == "libCore.dylib" || p.filename() == "libCore.so") {
+         rootlibdir = p.parent_path().c_str();
+         break;
+      }
    }
+
+#elif defined(_WIN32)
+
+   HMODULE hMods[1024];
+   DWORD cbNeeded;
+
+   if (EnumProcessModules(::GetCurrentProcess(), hMods, sizeof(hMods), &cbNeeded)) {
+      for (unsigned int i = 0; i < (cbNeeded / sizeof(HMODULE)); i++) {
+         char szModName[MAX_PATH];
+         if (GetModuleFileNameA(hMods[i], szModName, sizeof(szModName) / sizeof(char))) {
+            fs::path p(szModName);
+            if (p.filename() == "libCore.dll") { // ROOT Windows build
+               rootlibdir = p.parent_path().string().c_str();
+               break;
+            }
+         }
+      }
+   }
+
 #endif
+
+   return rootlibdir;
 }
 
 ////////////////////////////////////////////////////////////////////////////////